nv-tegra-mirror/linux-nv-oot
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nv-oot.git synced 2025-12-23 09:42:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

misc: mods: update from Perforce

Browse Source 
Bug 3992588

Change-Id: I04aad94aebef7a48d9afac528347fc50c219d888
Signed-off-by: Chris Dragan <kdragan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2872373
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: Carl Dong <carld@nvidia.com>
Reviewed-by: Sachin Nikam <snikam@nvidia.com>
GVS: Gerrit_Virtual_Submit <buildbot_gerritrpt@nvidia.com>
This commit is contained in:
Chris Dragan
2023-03-16 07:17:30 -07:00
committed by Laxman Dewangan
parent dbced7f181
commit cb0c45b6dd
7 changed files with 252 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/misc/mods/Makefile
View File

@@ -3,9 +3,6 @@
ccflags-y += -I$(srctree.nvidia)/include
ccflags-y += -DMODS_HAS_TEGRA
LINUX_VERSION := $(shell expr $(VERSION) \* 256 + $(PATCHLEVEL))
LINUX_VERSION_6_2 := $(shell expr 6 \* 256 + 2)
ifeq ($(CONFIG_TEGRA_OOT_MODULE),m)
CONFIG_MODS := m
endif

9
drivers/misc/mods/mods_bpmpipc.c
View File

@@ -26,9 +26,8 @@
#include <soc/tegra/ivc.h>
#include <soc/tegra/bpmp.h>
#include <linux/version.h>
#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
#include <linux/iosys-map.h>
#endif
@@ -86,7 +85,7 @@ static int bpmp_ipc_send(struct mods_client *client,
const void *data,
size_t sz)
{
#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
int err;
struct iosys_map ob;
@@ -119,7 +118,7 @@ static int bpmp_ipc_recv(struct mods_client *client,
u32 timeout_ms)
{
int err;
#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
struct iosys_map ib;
#else
const void *frame;
@@ -128,7 +127,7 @@ static int bpmp_ipc_recv(struct mods_client *client,
end = ktime_add_ms(ktime_get(), timeout_ms);
#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
do {
err = tegra_ivc_read_get_next_frame(ivc, &ib);
if (!err)

11
drivers/misc/mods/mods_internal.h
View File

@@ -198,6 +198,7 @@ struct SYS_MAP_MEMORY {
phys_addr_t phys_addr;
unsigned long virtual_addr;
unsigned long mapping_offs; /* mapped offset from the beginning of the allocation */
unsigned long mapping_length; /* how many bytes were mapped */
};
@@ -326,9 +327,6 @@ struct mods_priv {
/* Client structures */
struct mods_client clients[MODS_MAX_CLIENTS];
/* Mutex for guarding interrupt logic and PCI device enablement */
struct mutex mtx;
};
#ifdef MODS_HAS_POLL_T
@@ -382,13 +380,14 @@ struct mods_priv {
/* ** MODULE WIDE FUNCTIONS */
/* ************************************************************************* */
/* client */
struct mods_client *mods_client_from_id(u8 client_id);
int mods_is_client_enabled(u8 client_id);
/* irq */
void mods_init_irq(void);
void mods_cleanup_irq(void);
struct mutex *mods_get_irq_mutex(void);
struct mods_client *mods_alloc_client(void);
void mods_free_client_interrupts(struct mods_client *client);
void mods_free_client(u8 client_id);
POLL_TYPE mods_irq_event_check(u8 client_id);
/* mem */

140
drivers/misc/mods/mods_irq.c
View File

@@ -50,11 +50,13 @@
/*********************
* PRIVATE FUNCTIONS *
*********************/
static struct mods_priv mp;
/* Mutex for guarding interrupt logic and PCI device enablement */
static struct mutex irq_mtx;
struct mutex *mods_get_irq_mutex(void)
{
return &mp.mtx;
return &irq_mtx;
}
#ifdef CONFIG_PCI
@@ -65,7 +67,7 @@ int mods_enable_device(struct mods_client *client,
int err = OK;
struct en_dev_entry *dpriv = client->enabled_devices;
WARN_ON(!mutex_is_locked(&mp.mtx));
WARN_ON(!mutex_is_locked(&irq_mtx));
dpriv = pci_get_drvdata(dev);
if (dpriv) {
@@ -124,7 +126,7 @@ void mods_disable_device(struct mods_client *client,
{
struct en_dev_entry *dpriv = pci_get_drvdata(dev);
WARN_ON(!mutex_is_locked(&mp.mtx));
WARN_ON(!mutex_is_locked(&irq_mtx));
#ifdef MODS_HAS_SRIOV
if (dpriv && dpriv->num_vfs)
@@ -259,14 +261,9 @@ static const char *mods_irq_type_name(u8 irq_type)
}
#endif
static struct mods_client *client_from_id(u8 client_id)
{
return &mp.clients[client_id - 1];
}
static void wake_up_client(struct dev_irq_map *t)
{
struct mods_client *client = client_from_id(t->client_id);
struct mods_client *client = mods_client_from_id(t->client_id);
if (client)
wake_up_interruptible(&client->interrupt_event);
@@ -347,7 +344,7 @@ static irqreturn_t mods_irq_handle(int irq, void *data)
unsigned long flags = 0;
int recorded = false;
unsigned int irq_time = get_cur_time();
struct mods_client *client = client_from_id(t->client_id);
struct mods_client *client = mods_client_from_id(t->client_id);
spin_lock_irqsave(&client->irq_lock, flags);
@@ -385,12 +382,12 @@ static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq)
struct dev_irq_map *t = NULL;
struct dev_irq_map *next = NULL;
if (!test_bit(client_idx - 1, &mp.client_flags))
if (!mods_is_client_enabled(client_idx))
continue;
list_for_each_entry_safe(t,
next,
&client_from_id(client_idx)->irq_list,
&mods_client_from_id(client_idx)->irq_list,
list) {
if (t->apic_irq == irq) {
@@ -657,32 +654,19 @@ void mods_init_irq(void)
{
LOG_ENT();
memset(&mp, 0, sizeof(mp));
mutex_init(&irq_mtx);
mutex_init(&mp.mtx);
LOG_EXT();
}
void mods_cleanup_irq(void)
{
int i;
LOG_ENT();
for (i = 0; i < MODS_MAX_CLIENTS; i++) {
if (mp.client_flags & (1U << i))
mods_free_client(i + 1);
}
LOG_EXT();
}
POLL_TYPE mods_irq_event_check(u8 client_id)
{
struct irq_q_info *q = &client_from_id(client_id)->irq_queue;
unsigned int pos = (1U << (client_id - 1));
struct irq_q_info *q;
if (!(mp.client_flags & pos))
return POLLERR; /* irq has quit */
if (!mods_is_client_enabled(client_id))
return POLLERR; /* client has quit */
q = &mods_client_from_id(client_id)->irq_queue;
if (q->head != q->tail)
return POLLIN; /* irq generated */
@@ -690,51 +674,6 @@ POLL_TYPE mods_irq_event_check(u8 client_id)
return 0;
}
struct mods_client *mods_alloc_client(void)
{
unsigned int idx;
unsigned int max_clients = 1;
LOG_ENT();
if (mods_get_multi_instance() ||
(mods_get_access_token() != MODS_ACCESS_TOKEN_NONE))
max_clients = MODS_MAX_CLIENTS;
for (idx = 1; idx <= max_clients; idx++) {
if (!test_and_set_bit(idx - 1U, &mp.client_flags)) {
struct mods_client *client = client_from_id(idx);
memset(client, 0, sizeof(*client));
client->client_id = idx;
client->access_token = MODS_ACCESS_TOKEN_NONE;
atomic_set(&client->last_bad_dbdf, -1);
cl_debug(DEBUG_IOCTL,
"open client (bit mask 0x%lx)\n",
mp.client_flags);
mutex_init(&client->mtx);
spin_lock_init(&client->irq_lock);
init_waitqueue_head(&client->interrupt_event);
INIT_LIST_HEAD(&client->irq_list);
INIT_LIST_HEAD(&client->mem_alloc_list);
INIT_LIST_HEAD(&client->mem_map_list);
INIT_LIST_HEAD(&client->free_mem_list);
#if defined(CONFIG_PPC64)
INIT_LIST_HEAD(&client->ppc_tce_bypass_list);
INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list);
#endif
LOG_EXT();
return client;
}
}
LOG_EXT();
return NULL;
}
static int mods_free_irqs(struct mods_client *client,
struct pci_dev *dev)
{
@@ -746,7 +685,7 @@ static int mods_free_irqs(struct mods_client *client,
LOG_ENT();
if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
LOG_EXT();
return -EINTR;
}
@@ -754,7 +693,7 @@ static int mods_free_irqs(struct mods_client *client,
dpriv = pci_get_drvdata(dev);
if (!dpriv) {
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return OK;
}
@@ -767,7 +706,7 @@ static int mods_free_irqs(struct mods_client *client,
PCI_SLOT(dev->devfn),
PCI_FUNC(dev->devfn),
dpriv->client_id);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return -EINVAL;
}
@@ -823,7 +762,7 @@ static int mods_free_irqs(struct mods_client *client,
cl_debug(DEBUG_ISR_DETAILED, "irqs freed\n");
#endif
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return 0;
}
@@ -843,21 +782,6 @@ void mods_free_client_interrupts(struct mods_client *client)
LOG_EXT();
}
void mods_free_client(u8 client_id)
{
struct mods_client *client = client_from_id(client_id);
LOG_ENT();
memset(client, 0, sizeof(*client));
/* Indicate the client_id is free */
clear_bit((unsigned int)client_id - 1U, &mp.client_flags);
cl_debug(DEBUG_IOCTL, "closed client\n");
LOG_EXT();
}
#ifdef CONFIG_PCI
static int mods_allocate_irqs(struct mods_client *client,
struct pci_dev *dev,
@@ -1072,7 +996,7 @@ static int mods_register_pci_irq(struct mods_client *client,
return err;
}
if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
pci_dev_put(dev);
LOG_EXT();
return -EINTR;
@@ -1088,7 +1012,7 @@ static int mods_register_pci_irq(struct mods_client *client,
p->dev.device,
p->dev.function,
dpriv->client_id);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
pci_dev_put(dev);
LOG_EXT();
return -EBUSY;
@@ -1100,7 +1024,7 @@ static int mods_register_pci_irq(struct mods_client *client,
p->dev.bus,
p->dev.device,
p->dev.function);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
pci_dev_put(dev);
LOG_EXT();
return -EINVAL;
@@ -1112,7 +1036,7 @@ static int mods_register_pci_irq(struct mods_client *client,
if (err) {
cl_error("could not allocate irqs for irq_type %d\n",
irq_type);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
pci_dev_put(dev);
LOG_EXT();
return err;
@@ -1137,7 +1061,7 @@ static int mods_register_pci_irq(struct mods_client *client,
}
}
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
pci_dev_put(dev);
LOG_EXT();
return err;
@@ -1152,7 +1076,7 @@ static int mods_register_cpu_irq(struct mods_client *client,
LOG_ENT();
if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
LOG_EXT();
return -EINTR;
}
@@ -1160,7 +1084,7 @@ static int mods_register_cpu_irq(struct mods_client *client,
/* Determine if the interrupt is already hooked */
if (mods_lookup_cpu_irq(0, irq) == IRQ_FOUND) {
cl_error("CPU IRQ 0x%x has already been registered\n", irq);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return -EINVAL;
}
@@ -1168,7 +1092,7 @@ static int mods_register_cpu_irq(struct mods_client *client,
/* Register interrupt */
err = add_irq_map(client, NULL, p, irq, 0);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return err;
}
@@ -1208,7 +1132,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
irq = p->dev.bus;
if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
LOG_EXT();
return -EINTR;
}
@@ -1216,7 +1140,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
/* Determine if the interrupt is already hooked by this client */
if (mods_lookup_cpu_irq(client->client_id, irq) == IRQ_NOT_FOUND) {
cl_error("IRQ 0x%x not hooked, can't unhook\n", irq);
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return -EINVAL;
}
@@ -1226,7 +1150,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
if ((irq == del->apic_irq) && (del->dev == NULL)) {
if (del->type != p->type) {
cl_error("wrong IRQ type passed\n");
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return -EINVAL;
}
@@ -1239,7 +1163,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
}
}
mutex_unlock(&mp.mtx);
mutex_unlock(&irq_mtx);
LOG_EXT();
return OK;
}

206
drivers/misc/mods/mods_krnl.c
View File

@@ -29,7 +29,6 @@
#include <linux/random.h>
#include <linux/sched.h>
#include <linux/uaccess.h>
#include <linux/version.h>
#ifdef MODS_HAS_CONSOLE_LOCK
# include <linux/console.h>
# include <linux/kd.h>
@@ -42,6 +41,10 @@
# include <asm/msr.h>
#endif
#ifndef IS_BUILTIN
# define IS_BUILTIN(c) 0
#endif
/***********************************************************************
* mods_krnl_* functions, driver interfaces called by the Linux kernel *
***********************************************************************/
@@ -392,6 +395,81 @@ int mods_get_multi_instance(void)
return multi_instance > 0;
}
/*********************
* CLIENT MANAGEMENT *
*********************/
static struct mods_priv mp;
static struct mods_client *alloc_client(void)
{
unsigned int idx;
unsigned int max_clients = 1;
LOG_ENT();
if (mods_get_multi_instance() ||
(mods_get_access_token() != MODS_ACCESS_TOKEN_NONE))
max_clients = MODS_MAX_CLIENTS;
for (idx = 1; idx <= max_clients; idx++) {
if (!test_and_set_bit(idx - 1U, &mp.client_flags)) {
struct mods_client *client = mods_client_from_id(idx);
memset(client, 0, sizeof(*client));
client->client_id = idx;
client->access_token = MODS_ACCESS_TOKEN_NONE;
atomic_set(&client->last_bad_dbdf, -1);
cl_debug(DEBUG_IOCTL,
"open client (bit mask 0x%lx)\n",
mp.client_flags);
mutex_init(&client->mtx);
spin_lock_init(&client->irq_lock);
init_waitqueue_head(&client->interrupt_event);
INIT_LIST_HEAD(&client->irq_list);
INIT_LIST_HEAD(&client->mem_alloc_list);
INIT_LIST_HEAD(&client->mem_map_list);
INIT_LIST_HEAD(&client->free_mem_list);
#if defined(CONFIG_PPC64)
INIT_LIST_HEAD(&client->ppc_tce_bypass_list);
INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list);
#endif
LOG_EXT();
return client;
}
}
LOG_EXT();
return NULL;
}
static void free_client(u8 client_id)
{
struct mods_client *client = mods_client_from_id(client_id);
LOG_ENT();
memset(client, 0, sizeof(*client));
/* Indicate the client_id is free */
clear_bit((unsigned int)client_id - 1U, &mp.client_flags);
cl_debug(DEBUG_IOCTL, "closed client\n");
LOG_EXT();
}
struct mods_client *mods_client_from_id(u8 client_id)
{
return &mp.clients[client_id - 1];
}
int mods_is_client_enabled(u8 client_id)
{
return test_bit(client_id - 1, &mp.client_flags);
}
u32 mods_get_access_token(void)
{
return access_token;
@@ -404,7 +482,9 @@ static int validate_client(struct mods_client *client)
return false;
}
if (client->client_id < 1 || client->client_id > MODS_MAX_CLIENTS) {
if (client->client_id < 1 ||
client->client_id > MODS_MAX_CLIENTS ||
!mods_is_client_enabled(client->client_id)) {
cl_error("invalid client id\n");
return false;
}
@@ -446,6 +526,8 @@ static int __init mods_init_module(void)
LOG_ENT();
memset(&mp, 0, sizeof(mp));
mods_init_irq();
rc = misc_register(&mods_dev);
@@ -488,12 +570,10 @@ static int __init mods_init_module(void)
#endif
#endif
#if defined(CONFIG_ARM_FFA_TRANSPORT)
#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE)
#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
rc = mods_ffa_abi_register();
if (rc < 0)
mods_warning_printk("error on mods_ffa_abi_register returned %d\n", rc);
#endif
#endif
mods_info_printk("*** WARNING: DIAGNOSTIC DRIVER LOADED ***\n");
@@ -510,13 +590,18 @@ static int __init mods_init_module(void)
static void __exit mods_exit_module(void)
{
int i;
LOG_ENT();
mods_exit_dmabuf();
mods_remove_debugfs();
mods_cleanup_irq();
for (i = 0; i < MODS_MAX_CLIENTS; i++) {
if (mp.client_flags & (1U << i))
free_client(i + 1);
}
#if defined(MODS_HAS_TEGRA)
#if defined(CONFIG_DMA_ENGINE)
@@ -536,12 +621,9 @@ static void __exit mods_exit_module(void)
mods_shutdown_clock_api();
#endif
#if defined(CONFIG_ARM_FFA_TRANSPORT)
#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE)
#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
mods_ffa_abi_unregister();
#endif
#endif
mods_info_printk("driver unloaded\n");
LOG_EXT();
}
@@ -615,6 +697,7 @@ static int register_mapping(struct mods_client *client,
struct MODS_MEM_INFO *p_mem_info,
phys_addr_t phys_addr,
unsigned long virtual_address,
unsigned long mapping_offs,
unsigned long mapping_length)
{
struct SYS_MAP_MEMORY *p_map_mem;
@@ -630,6 +713,7 @@ static int register_mapping(struct mods_client *client,
p_map_mem->phys_addr = phys_addr;
p_map_mem->virtual_addr = virtual_address;
p_map_mem->mapping_offs = mapping_offs;
p_map_mem->mapping_length = mapping_length;
p_map_mem->p_mem_info = p_mem_info;
@@ -697,7 +781,7 @@ static void unregister_all_mappings(struct mods_client *client)
LOG_EXT();
}
static pgprot_t mods_get_prot(struct mods_client *client,
static pgprot_t get_prot(struct mods_client *client,
u8 mem_type,
pgprot_t prot)
{
@@ -717,17 +801,33 @@ static pgprot_t mods_get_prot(struct mods_client *client,
}
}
static pgprot_t get_prot_for_range(struct mods_client *client,
static int get_prot_for_range(struct mods_client *client,
phys_addr_t phys_addr,
unsigned long size,
pgprot_t prot)
pgprot_t *prot)
{
if ((phys_addr == client->mem_type.phys_addr) &&
(size == client->mem_type.size)) {
const phys_addr_t req_end = phys_addr + size;
const phys_addr_t range_begin = client->mem_type.phys_addr;
const phys_addr_t range_end = range_begin + client->mem_type.size;
return mods_get_prot(client, client->mem_type.type, prot);
/* Check overlap with set memory type range */
if (phys_addr < range_end && req_end > range_begin) {
/* Check if requested range is completely inside */
if (likely(phys_addr >= range_begin && req_end <= range_end)) {
*prot = get_prot(client, client->mem_type.type, *prot);
return 0;
}
return prot;
cl_error("mmap range [0x%llx, 0x%llx] does not match set memory type range [0x%llx, 0x%llx]\n",
(unsigned long long)phys_addr,
(unsigned long long)req_end,
(unsigned long long)range_begin,
(unsigned long long)range_end);
return -EINVAL;
}
return 0;
}
const char *mods_get_prot_str(u8 mem_type)
@@ -751,13 +851,17 @@ static const char *get_prot_str_for_range(struct mods_client *client,
phys_addr_t phys_addr,
unsigned long size)
{
if ((phys_addr == client->mem_type.phys_addr) &&
(size == client->mem_type.size)) {
const phys_addr_t req_end = phys_addr + size;
const phys_addr_t range_begin = client->mem_type.phys_addr;
const phys_addr_t range_end = range_begin + client->mem_type.size;
/* Check overlap with set memory type range */
if (phys_addr < range_end && req_end > range_begin)
return mods_get_prot_str(client->mem_type.type);
}
return "default";
}
/********************
* PCI ERROR FUNCTIONS *
********************/
@@ -807,8 +911,9 @@ static void mods_krnl_vma_open(struct vm_area_struct *vma)
LOG_ENT();
mods_debug_printk(DEBUG_MEM_DETAILED,
"open vma, virt 0x%lx, phys 0x%llx\n",
"open vma, virt 0x%lx, size 0x%lx, phys 0x%llx\n",
vma->vm_start,
vma->vm_end - vma->vm_start,
(unsigned long long)vma->vm_pgoff << PAGE_SHIFT);
priv = vma->vm_private_data;
@@ -829,16 +934,15 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma)
struct mods_client *client = priv->client;
struct SYS_MAP_MEMORY *p_map_mem;
if (unlikely(mutex_lock_interruptible(&client->mtx))) {
LOG_EXT();
return;
}
mutex_lock(&client->mtx);
/* we need to unregister the mapping */
p_map_mem = find_mapping(client, vma->vm_start);
if (p_map_mem)
unregister_mapping(client, p_map_mem);
mutex_unlock(&client->mtx);
mods_debug_printk(DEBUG_MEM_DETAILED,
"closed vma, virt 0x%lx\n",
vma->vm_start);
@@ -847,8 +951,6 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma)
kfree(priv);
atomic_dec(&client->num_allocs);
mutex_unlock(&client->mtx);
}
LOG_EXT();
@@ -877,9 +979,12 @@ static int mods_krnl_vma_access(struct vm_area_struct *vma,
client = priv->client;
cl_debug(DEBUG_MEM_DETAILED,
"access vma, virt 0x%lx, phys 0x%llx\n",
"access vma [virt 0x%lx, size 0x%lx, phys 0x%llx] at virt 0x%lx, len 0x%x\n",
vma->vm_start,
(unsigned long long)vma->vm_pgoff << PAGE_SHIFT);
vma->vm_end - vma->vm_start,
(unsigned long long)vma->vm_pgoff << PAGE_SHIFT,
addr,
len);
if (unlikely(mutex_lock_interruptible(&client->mtx))) {
LOG_EXT();
@@ -891,12 +996,14 @@ static int mods_krnl_vma_access(struct vm_area_struct *vma,
if (unlikely(!p_map_mem || addr < p_map_mem->virtual_addr ||
addr + len > p_map_mem->virtual_addr +
p_map_mem->mapping_length)) {
cl_error("mapped range [virt 0x%lx, size 0x%x] does not match vma [virt 0x%lx, size 0x%lx]\n",
addr, len, vma->vm_start, vma->vm_end - vma->vm_start);
mutex_unlock(&client->mtx);
LOG_EXT();
return -ENOMEM;
}
map_offs = addr - vma->vm_start;
map_offs = addr - vma->vm_start + p_map_mem->mapping_offs;
if (p_map_mem->p_mem_info) {
struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info;
@@ -985,7 +1092,7 @@ static int mods_krnl_open(struct inode *ip, struct file *fp)
LOG_ENT();
client = mods_alloc_client();
client = alloc_client();
if (client == NULL) {
mods_error_printk("too many clients\n");
LOG_EXT();
@@ -1063,7 +1170,7 @@ static int mods_krnl_close(struct inode *ip, struct file *fp)
client->work_queue = NULL;
}
mods_free_client(client_id);
free_client(client_id);
pr_info("mods [%d]: driver closed\n", client_id);
@@ -1136,13 +1243,15 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma)
mods_krnl_vma_open(vma);
if (unlikely(mutex_lock_interruptible(&client->mtx)))
err = -EINTR;
else {
err = mutex_lock_interruptible(&client->mtx);
if (likely(!err)) {
err = mods_krnl_map_inner(client, vma);
mutex_unlock(&client->mtx);
}
if (unlikely(err))
mods_krnl_vma_close(vma);
LOG_EXT();
return err;
}
@@ -1152,6 +1261,7 @@ static int map_system_mem(struct mods_client *client,
struct MODS_MEM_INFO *p_mem_info)
{
struct scatterlist *sg = NULL;
const char *cache_str = mods_get_prot_str(p_mem_info->cache_type);
const phys_addr_t req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
phys_addr_t reg_pa = 0;
const unsigned long vma_size = vma->vm_end - vma->vm_start;
@@ -1161,7 +1271,7 @@ static int map_system_mem(struct mods_client *client,
const u32 num_chunks = get_num_chunks(p_mem_info);
u32 map_chunks;
u32 i = 0;
const pgprot_t prot = mods_get_prot(client,
const pgprot_t prot = get_prot(client,
p_mem_info->cache_type,
vma->vm_page_prot);
@@ -1202,6 +1312,7 @@ static int map_system_mem(struct mods_client *client,
map_pa += aoffs;
map_size -= aoffs;
skip_size += aoffs;
reg_pa = chunk_pa;
}
@@ -1209,11 +1320,12 @@ static int map_system_mem(struct mods_client *client,
map_size = (unsigned int)size_to_map;
cl_debug(DEBUG_MEM_DETAILED,
"remap va 0x%lx pfn 0x%lx size 0x%x pages %u\n",
"remap va 0x%lx pfn 0x%lx size 0x%x pages %u %s\n",
map_va,
(unsigned long)(map_pa >> PAGE_SHIFT),
map_size,
map_size >> PAGE_SHIFT);
map_size >> PAGE_SHIFT,
cache_str);
if (remap_pfn_range(vma,
map_va,
@@ -1234,6 +1346,7 @@ static int map_system_mem(struct mods_client *client,
p_mem_info,
reg_pa,
vma->vm_start,
skip_size,
vma_size);
return OK;
@@ -1245,6 +1358,8 @@ static int map_device_mem(struct mods_client *client,
/* device memory */
const phys_addr_t req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
const unsigned long vma_size = vma->vm_end - vma->vm_start;
pgprot_t prot = vma->vm_page_prot;
int err;
cl_debug(DEBUG_MEM,
"map dev: phys 0x%llx, virt 0x%lx, size 0x%lx, %s\n",
@@ -1253,13 +1368,15 @@ static int map_device_mem(struct mods_client *client,
vma_size,
get_prot_str_for_range(client, req_pa, vma_size));
if (io_remap_pfn_range(
vma,
err = get_prot_for_range(client, req_pa, vma_size, &prot);
if (unlikely(err))
return err;
if (unlikely(io_remap_pfn_range(vma,
vma->vm_start,
vma->vm_pgoff,
vma_size,
get_prot_for_range(client, req_pa, vma_size,
vma->vm_page_prot))) {
prot))) {
cl_error("failed to map device memory\n");
return -EAGAIN;
}
@@ -1268,6 +1385,7 @@ static int map_device_mem(struct mods_client *client,
NULL,
req_pa,
vma->vm_start,
0,
vma_size);
return OK;
@@ -2664,12 +2782,10 @@ static long mods_krnl_ioctl(struct file *fp,
#endif
#if defined(CONFIG_ARM_FFA_TRANSPORT)
#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE)
#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
case MODS_ESC_FFA_CMD:
MODS_IOCTL(MODS_ESC_FFA_CMD, esc_mods_arm_ffa_cmd, MODS_FFA_PARAMS);
break;
#endif
#endif
case MODS_ESC_ACQUIRE_ACCESS_TOKEN:

169
drivers/misc/mods/mods_mem.c
View File

@@ -22,6 +22,7 @@
#include <linux/bitops.h>
#include <linux/pagemap.h>
#include <linux/sched.h>
#if defined(MODS_HAS_SET_DMA_MASK)
#include <linux/dma-mapping.h>
@@ -412,7 +413,6 @@ static int dma_map_to_default_dev(struct mods_client *client,
#ifdef CONFIG_ARM64
static void clear_contiguous_cache(struct mods_client *client,
u64 virt_start,
u64 phys_start,
u32 size);
#endif
@@ -438,7 +438,6 @@ static int setup_cache_attr(struct mods_client *client,
#ifdef CONFIG_ARM64
clear_contiguous_cache(client,
(u64)(size_t)ptr,
sg_phys(sg) + offs,
PAGE_SIZE);
#else
if (p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE)
@@ -458,6 +457,9 @@ static int setup_cache_attr(struct mods_client *client,
* even for chunks where we haven't change them.
*/
mark_chunk_wc(p_mem_info, ichunk);
/* Avoid superficial lockups */
cond_resched();
}
}
@@ -2077,7 +2079,6 @@ int esc_mods_virtual_to_phys(struct mods_client *client,
list_for_each(iter, head) {
struct SYS_MAP_MEMORY *p_map_mem;
u64 begin, end;
u64 phys_offs;
p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list);
@@ -2104,14 +2105,9 @@ int esc_mods_virtual_to_phys(struct mods_client *client,
return OK;
}
if (get_alloc_offset(p_map_mem->p_mem_info,
p_map_mem->phys_addr,
&phys_offs) != OK)
break;
range.memory_handle =
(u64)(size_t)p_map_mem->p_mem_info;
range.offset = virt_offs + phys_offs;
range.offset = virt_offs + p_map_mem->mapping_offs;
mutex_unlock(&client->mtx);
@@ -2194,10 +2190,7 @@ int esc_mods_phys_to_virtual(struct mods_client *client,
continue;
/* offset from the beginning of the mapping */
if (get_alloc_offset(p_map_mem->p_mem_info,
p_map_mem->phys_addr,
&map_offset))
continue;
map_offset = p_map_mem->mapping_offs;
if ((offset >= map_offset) &&
(offset < map_offset + p_map_mem->mapping_length)) {
@@ -2474,7 +2467,6 @@ failed:
#ifdef CONFIG_ARM64
static void clear_contiguous_cache(struct mods_client *client,
u64 virt_start,
u64 phys_start,
u32 size)
{
u64 end = virt_start + size;
@@ -2483,16 +2475,16 @@ static void clear_contiguous_cache(struct mods_client *client,
static u32 d_line_shift;
if (!d_line_shift) {
#if KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION
const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
#if KERNEL_VERSION(6, 0, 0) <= MODS_KERNEL_VERSION
const int field = CTR_EL0_DminLine_SHIFT;
#else
const int field = CTR_DMINLINE_SHIFT;
#endif
const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
d_line_shift =
cpuid_feature_extract_unsigned_field(ctr_el0, field);
cpuid_feature_extract_unsigned_field(ctr_el0, CTR_EL0_DminLine_SHIFT);
#elif KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION
const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
d_line_shift =
cpuid_feature_extract_unsigned_field(ctr_el0, CTR_DMINLINE_SHIFT);
#else
d_line_shift = 4; /* Fallback for kernel 5.9 or older */
#endif
@@ -2502,91 +2494,15 @@ static void clear_contiguous_cache(struct mods_client *client,
cur = virt_start & ~(d_size - 1);
do {
asm volatile("dc civac, %0" : : "r" (cur) : "memory");
/* Avoid superficial lockups */
cond_resched();
} while (cur += d_size, cur < end);
asm volatile("dsb sy" : : : "memory");
cl_debug(DEBUG_MEM_DETAILED,
"clear cache virt 0x%llx phys 0x%llx size 0x%x\n",
virt_start, phys_start, size);
}
static void clear_entry_cache_mappings(struct mods_client *client,
struct SYS_MAP_MEMORY *p_map_mem,
u64 virt_offs,
u64 virt_offs_end)
{
struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info;
struct scatterlist *sg;
u64 cur_vo = p_map_mem->virtual_addr;
u32 num_chunks;
u32 i;
if (!p_mem_info)
return;
if (p_mem_info->cache_type != MODS_ALLOC_CACHED)
return;
num_chunks = get_num_chunks(p_mem_info);
for_each_sg(p_mem_info->sg, sg, num_chunks, i) {
u32 chunk_offs = 0;
u32 chunk_offs_end = sg->length;
u64 cur_vo_end = cur_vo + chunk_offs_end;
if (virt_offs_end <= cur_vo)
break;
if (virt_offs >= cur_vo_end) {
cur_vo = cur_vo_end;
continue;
}
if (cur_vo < virt_offs)
chunk_offs = (u32)(virt_offs - cur_vo);
if (virt_offs_end < cur_vo_end)
chunk_offs_end -= (u32)(cur_vo_end - virt_offs_end);
cl_debug(DEBUG_MEM_DETAILED,
"clear cache %p [%u]\n",
p_mem_info,
i);
while (chunk_offs < chunk_offs_end) {
u32 i_page = chunk_offs >> PAGE_SHIFT;
u32 page_offs = chunk_offs - (i_page << PAGE_SHIFT);
u64 page_va =
(u64)(size_t)MODS_KMAP(sg_page(sg) + i_page);
u64 clear_va = page_va + page_offs;
u64 clear_pa = sg_phys(sg) + chunk_offs;
u32 clear_size = PAGE_SIZE - page_offs;
u64 remaining = chunk_offs_end - chunk_offs;
if (likely(page_va)) {
if ((u64)clear_size > remaining)
clear_size = (u32)remaining;
cl_debug(DEBUG_MEM_DETAILED,
"clear page %u, chunk offs 0x%x, page va 0x%llx\n",
i_page,
chunk_offs,
page_va);
clear_contiguous_cache(client,
clear_va,
clear_pa,
clear_size);
kunmap((void *)(size_t)page_va);
} else {
cl_error("kmap failed\n");
}
chunk_offs += clear_size;
}
cur_vo = cur_vo_end;
}
"flush cache virt 0x%llx size 0x%x\n",
virt_start, size);
}
int esc_mods_flush_cpu_cache_range(struct mods_client *client,
@@ -2619,45 +2535,34 @@ int esc_mods_flush_cpu_cache_range(struct mods_client *client,
}
head = &client->mem_map_list;
err = -EINVAL;
list_for_each(iter, head) {
struct SYS_MAP_MEMORY *p_map_mem
= list_entry(iter, struct SYS_MAP_MEMORY, list);
u64 mapped_va = p_map_mem->virtual_addr;
const u64 mapped_va = p_map_mem->virtual_addr;
const u64 mapped_end = mapped_va + p_map_mem->mapping_length;
const u64 flush_start = p->virt_addr_start < mapped_va ? mapped_va
: p->virt_addr_start;
const u64 flush_end = p->virt_addr_end > mapped_end ? mapped_end
: p->virt_addr_end;
/* Note: mapping end points to the first address of next range*/
u64 mapping_end = mapped_va + p_map_mem->mapping_length;
if (flush_start >= flush_end)
continue;
int start_on_page = p->virt_addr_start >= mapped_va
&& p->virt_addr_start < mapping_end;
int start_before_page = p->virt_addr_start < mapped_va;
int end_on_page = p->virt_addr_end >= mapped_va
&& p->virt_addr_end < mapping_end;
int end_after_page = p->virt_addr_end >= mapping_end;
u64 virt_start = p->virt_addr_start;
/* Kernel expects end to point to the first address of next
* range
*/
u64 virt_end = p->virt_addr_end + 1;
if ((start_on_page || start_before_page)
&& (end_on_page || end_after_page)) {
if (!start_on_page)
virt_start = p_map_mem->virtual_addr;
if (!end_on_page)
virt_end = mapping_end;
clear_entry_cache_mappings(client,
p_map_mem,
virt_start,
virt_end);
}
clear_contiguous_cache(client, flush_start, flush_end - flush_start);
err = OK;
}
mutex_unlock(&client->mtx);
if (err)
cl_error("va range 0x%lx..0x%lx not flushed\n",
(unsigned long)p->virt_addr_start,
(unsigned long)p->virt_addr_end);
LOG_EXT();
return OK;
return err;
}
#endif /* CONFIG_ARM64 */

2
include/uapi/misc/mods.h
View File

@@ -25,7 +25,7 @@
/* Driver version */
#define MODS_DRIVER_VERSION_MAJOR 4
#define MODS_DRIVER_VERSION_MINOR 13
#define MODS_DRIVER_VERSION_MINOR 15
#define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \
((MODS_DRIVER_VERSION_MINOR / 10) << 4) | \
(MODS_DRIVER_VERSION_MINOR % 10))