diff --git a/drivers/misc/mods/Makefile b/drivers/misc/mods/Makefile index 8b9a22bf..4d9132a0 100644 --- a/drivers/misc/mods/Makefile +++ b/drivers/misc/mods/Makefile @@ -3,9 +3,6 @@ ccflags-y += -I$(srctree.nvidia)/include ccflags-y += -DMODS_HAS_TEGRA -LINUX_VERSION := $(shell expr $(VERSION) \* 256 + $(PATCHLEVEL)) -LINUX_VERSION_6_2 := $(shell expr 6 \* 256 + 2) - ifeq ($(CONFIG_TEGRA_OOT_MODULE),m) CONFIG_MODS := m endif diff --git a/drivers/misc/mods/mods_bpmpipc.c b/drivers/misc/mods/mods_bpmpipc.c index eba4cad5..f4a3b7a0 100644 --- a/drivers/misc/mods/mods_bpmpipc.c +++ b/drivers/misc/mods/mods_bpmpipc.c @@ -26,9 +26,8 @@ #include #include -#include -#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE) +#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) #include #endif @@ -86,7 +85,7 @@ static int bpmp_ipc_send(struct mods_client *client, const void *data, size_t sz) { -#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE) +#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) int err; struct iosys_map ob; @@ -119,7 +118,7 @@ static int bpmp_ipc_recv(struct mods_client *client, u32 timeout_ms) { int err; -#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE) +#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) struct iosys_map ib; #else const void *frame; @@ -128,7 +127,7 @@ static int bpmp_ipc_recv(struct mods_client *client, end = ktime_add_ms(ktime_get(), timeout_ms); -#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE) +#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION) do { err = tegra_ivc_read_get_next_frame(ivc, &ib); if (!err) diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h index 55de8672..a25c8cf7 100644 --- a/drivers/misc/mods/mods_internal.h +++ b/drivers/misc/mods/mods_internal.h @@ -198,6 +198,7 @@ struct SYS_MAP_MEMORY { phys_addr_t phys_addr; unsigned long virtual_addr; + unsigned long mapping_offs; /* mapped offset from the beginning of the allocation */ unsigned long mapping_length; /* how many bytes were mapped */ }; @@ -326,9 +327,6 @@ struct mods_priv { /* Client structures */ struct mods_client clients[MODS_MAX_CLIENTS]; - - /* Mutex for guarding interrupt logic and PCI device enablement */ - struct mutex mtx; }; #ifdef MODS_HAS_POLL_T @@ -382,13 +380,14 @@ struct mods_priv { /* ** MODULE WIDE FUNCTIONS */ /* ************************************************************************* */ +/* client */ +struct mods_client *mods_client_from_id(u8 client_id); +int mods_is_client_enabled(u8 client_id); + /* irq */ void mods_init_irq(void); -void mods_cleanup_irq(void); struct mutex *mods_get_irq_mutex(void); -struct mods_client *mods_alloc_client(void); void mods_free_client_interrupts(struct mods_client *client); -void mods_free_client(u8 client_id); POLL_TYPE mods_irq_event_check(u8 client_id); /* mem */ diff --git a/drivers/misc/mods/mods_irq.c b/drivers/misc/mods/mods_irq.c index d34c318a..cfeb2e28 100644 --- a/drivers/misc/mods/mods_irq.c +++ b/drivers/misc/mods/mods_irq.c @@ -50,11 +50,13 @@ /********************* * PRIVATE FUNCTIONS * *********************/ -static struct mods_priv mp; + +/* Mutex for guarding interrupt logic and PCI device enablement */ +static struct mutex irq_mtx; struct mutex *mods_get_irq_mutex(void) { - return &mp.mtx; + return &irq_mtx; } #ifdef CONFIG_PCI @@ -65,7 +67,7 @@ int mods_enable_device(struct mods_client *client, int err = OK; struct en_dev_entry *dpriv = client->enabled_devices; - WARN_ON(!mutex_is_locked(&mp.mtx)); + WARN_ON(!mutex_is_locked(&irq_mtx)); dpriv = pci_get_drvdata(dev); if (dpriv) { @@ -124,7 +126,7 @@ void mods_disable_device(struct mods_client *client, { struct en_dev_entry *dpriv = pci_get_drvdata(dev); - WARN_ON(!mutex_is_locked(&mp.mtx)); + WARN_ON(!mutex_is_locked(&irq_mtx)); #ifdef MODS_HAS_SRIOV if (dpriv && dpriv->num_vfs) @@ -259,14 +261,9 @@ static const char *mods_irq_type_name(u8 irq_type) } #endif -static struct mods_client *client_from_id(u8 client_id) -{ - return &mp.clients[client_id - 1]; -} - static void wake_up_client(struct dev_irq_map *t) { - struct mods_client *client = client_from_id(t->client_id); + struct mods_client *client = mods_client_from_id(t->client_id); if (client) wake_up_interruptible(&client->interrupt_event); @@ -347,7 +344,7 @@ static irqreturn_t mods_irq_handle(int irq, void *data) unsigned long flags = 0; int recorded = false; unsigned int irq_time = get_cur_time(); - struct mods_client *client = client_from_id(t->client_id); + struct mods_client *client = mods_client_from_id(t->client_id); spin_lock_irqsave(&client->irq_lock, flags); @@ -385,12 +382,12 @@ static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq) struct dev_irq_map *t = NULL; struct dev_irq_map *next = NULL; - if (!test_bit(client_idx - 1, &mp.client_flags)) + if (!mods_is_client_enabled(client_idx)) continue; list_for_each_entry_safe(t, next, - &client_from_id(client_idx)->irq_list, + &mods_client_from_id(client_idx)->irq_list, list) { if (t->apic_irq == irq) { @@ -657,32 +654,19 @@ void mods_init_irq(void) { LOG_ENT(); - memset(&mp, 0, sizeof(mp)); + mutex_init(&irq_mtx); - mutex_init(&mp.mtx); - - LOG_EXT(); -} - -void mods_cleanup_irq(void) -{ - int i; - - LOG_ENT(); - for (i = 0; i < MODS_MAX_CLIENTS; i++) { - if (mp.client_flags & (1U << i)) - mods_free_client(i + 1); - } LOG_EXT(); } POLL_TYPE mods_irq_event_check(u8 client_id) { - struct irq_q_info *q = &client_from_id(client_id)->irq_queue; - unsigned int pos = (1U << (client_id - 1)); + struct irq_q_info *q; - if (!(mp.client_flags & pos)) - return POLLERR; /* irq has quit */ + if (!mods_is_client_enabled(client_id)) + return POLLERR; /* client has quit */ + + q = &mods_client_from_id(client_id)->irq_queue; if (q->head != q->tail) return POLLIN; /* irq generated */ @@ -690,51 +674,6 @@ POLL_TYPE mods_irq_event_check(u8 client_id) return 0; } -struct mods_client *mods_alloc_client(void) -{ - unsigned int idx; - unsigned int max_clients = 1; - - LOG_ENT(); - - if (mods_get_multi_instance() || - (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE)) - max_clients = MODS_MAX_CLIENTS; - - for (idx = 1; idx <= max_clients; idx++) { - if (!test_and_set_bit(idx - 1U, &mp.client_flags)) { - struct mods_client *client = client_from_id(idx); - - memset(client, 0, sizeof(*client)); - client->client_id = idx; - client->access_token = MODS_ACCESS_TOKEN_NONE; - atomic_set(&client->last_bad_dbdf, -1); - - cl_debug(DEBUG_IOCTL, - "open client (bit mask 0x%lx)\n", - mp.client_flags); - - mutex_init(&client->mtx); - spin_lock_init(&client->irq_lock); - init_waitqueue_head(&client->interrupt_event); - INIT_LIST_HEAD(&client->irq_list); - INIT_LIST_HEAD(&client->mem_alloc_list); - INIT_LIST_HEAD(&client->mem_map_list); - INIT_LIST_HEAD(&client->free_mem_list); -#if defined(CONFIG_PPC64) - INIT_LIST_HEAD(&client->ppc_tce_bypass_list); - INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list); -#endif - - LOG_EXT(); - return client; - } - } - - LOG_EXT(); - return NULL; -} - static int mods_free_irqs(struct mods_client *client, struct pci_dev *dev) { @@ -746,7 +685,7 @@ static int mods_free_irqs(struct mods_client *client, LOG_ENT(); - if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + if (unlikely(mutex_lock_interruptible(&irq_mtx))) { LOG_EXT(); return -EINTR; } @@ -754,7 +693,7 @@ static int mods_free_irqs(struct mods_client *client, dpriv = pci_get_drvdata(dev); if (!dpriv) { - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return OK; } @@ -767,7 +706,7 @@ static int mods_free_irqs(struct mods_client *client, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), dpriv->client_id); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return -EINVAL; } @@ -823,7 +762,7 @@ static int mods_free_irqs(struct mods_client *client, cl_debug(DEBUG_ISR_DETAILED, "irqs freed\n"); #endif - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return 0; } @@ -843,21 +782,6 @@ void mods_free_client_interrupts(struct mods_client *client) LOG_EXT(); } -void mods_free_client(u8 client_id) -{ - struct mods_client *client = client_from_id(client_id); - - LOG_ENT(); - - memset(client, 0, sizeof(*client)); - - /* Indicate the client_id is free */ - clear_bit((unsigned int)client_id - 1U, &mp.client_flags); - - cl_debug(DEBUG_IOCTL, "closed client\n"); - LOG_EXT(); -} - #ifdef CONFIG_PCI static int mods_allocate_irqs(struct mods_client *client, struct pci_dev *dev, @@ -1072,7 +996,7 @@ static int mods_register_pci_irq(struct mods_client *client, return err; } - if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + if (unlikely(mutex_lock_interruptible(&irq_mtx))) { pci_dev_put(dev); LOG_EXT(); return -EINTR; @@ -1088,7 +1012,7 @@ static int mods_register_pci_irq(struct mods_client *client, p->dev.device, p->dev.function, dpriv->client_id); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); pci_dev_put(dev); LOG_EXT(); return -EBUSY; @@ -1100,7 +1024,7 @@ static int mods_register_pci_irq(struct mods_client *client, p->dev.bus, p->dev.device, p->dev.function); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); pci_dev_put(dev); LOG_EXT(); return -EINVAL; @@ -1112,7 +1036,7 @@ static int mods_register_pci_irq(struct mods_client *client, if (err) { cl_error("could not allocate irqs for irq_type %d\n", irq_type); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); pci_dev_put(dev); LOG_EXT(); return err; @@ -1137,7 +1061,7 @@ static int mods_register_pci_irq(struct mods_client *client, } } - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); pci_dev_put(dev); LOG_EXT(); return err; @@ -1152,7 +1076,7 @@ static int mods_register_cpu_irq(struct mods_client *client, LOG_ENT(); - if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + if (unlikely(mutex_lock_interruptible(&irq_mtx))) { LOG_EXT(); return -EINTR; } @@ -1160,7 +1084,7 @@ static int mods_register_cpu_irq(struct mods_client *client, /* Determine if the interrupt is already hooked */ if (mods_lookup_cpu_irq(0, irq) == IRQ_FOUND) { cl_error("CPU IRQ 0x%x has already been registered\n", irq); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return -EINVAL; } @@ -1168,7 +1092,7 @@ static int mods_register_cpu_irq(struct mods_client *client, /* Register interrupt */ err = add_irq_map(client, NULL, p, irq, 0); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return err; } @@ -1208,7 +1132,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client, irq = p->dev.bus; - if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + if (unlikely(mutex_lock_interruptible(&irq_mtx))) { LOG_EXT(); return -EINTR; } @@ -1216,7 +1140,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client, /* Determine if the interrupt is already hooked by this client */ if (mods_lookup_cpu_irq(client->client_id, irq) == IRQ_NOT_FOUND) { cl_error("IRQ 0x%x not hooked, can't unhook\n", irq); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return -EINVAL; } @@ -1226,7 +1150,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client, if ((irq == del->apic_irq) && (del->dev == NULL)) { if (del->type != p->type) { cl_error("wrong IRQ type passed\n"); - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return -EINVAL; } @@ -1239,7 +1163,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client, } } - mutex_unlock(&mp.mtx); + mutex_unlock(&irq_mtx); LOG_EXT(); return OK; } diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c index 1820724f..135d7545 100644 --- a/drivers/misc/mods/mods_krnl.c +++ b/drivers/misc/mods/mods_krnl.c @@ -29,7 +29,6 @@ #include #include #include -#include #ifdef MODS_HAS_CONSOLE_LOCK # include # include @@ -42,6 +41,10 @@ # include #endif +#ifndef IS_BUILTIN +# define IS_BUILTIN(c) 0 +#endif + /*********************************************************************** * mods_krnl_* functions, driver interfaces called by the Linux kernel * ***********************************************************************/ @@ -392,6 +395,81 @@ int mods_get_multi_instance(void) return multi_instance > 0; } +/********************* + * CLIENT MANAGEMENT * + *********************/ +static struct mods_priv mp; + +static struct mods_client *alloc_client(void) +{ + unsigned int idx; + unsigned int max_clients = 1; + + LOG_ENT(); + + if (mods_get_multi_instance() || + (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE)) + max_clients = MODS_MAX_CLIENTS; + + for (idx = 1; idx <= max_clients; idx++) { + if (!test_and_set_bit(idx - 1U, &mp.client_flags)) { + struct mods_client *client = mods_client_from_id(idx); + + memset(client, 0, sizeof(*client)); + client->client_id = idx; + client->access_token = MODS_ACCESS_TOKEN_NONE; + atomic_set(&client->last_bad_dbdf, -1); + + cl_debug(DEBUG_IOCTL, + "open client (bit mask 0x%lx)\n", + mp.client_flags); + + mutex_init(&client->mtx); + spin_lock_init(&client->irq_lock); + init_waitqueue_head(&client->interrupt_event); + INIT_LIST_HEAD(&client->irq_list); + INIT_LIST_HEAD(&client->mem_alloc_list); + INIT_LIST_HEAD(&client->mem_map_list); + INIT_LIST_HEAD(&client->free_mem_list); +#if defined(CONFIG_PPC64) + INIT_LIST_HEAD(&client->ppc_tce_bypass_list); + INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list); +#endif + + LOG_EXT(); + return client; + } + } + + LOG_EXT(); + return NULL; +} + +static void free_client(u8 client_id) +{ + struct mods_client *client = mods_client_from_id(client_id); + + LOG_ENT(); + + memset(client, 0, sizeof(*client)); + + /* Indicate the client_id is free */ + clear_bit((unsigned int)client_id - 1U, &mp.client_flags); + + cl_debug(DEBUG_IOCTL, "closed client\n"); + LOG_EXT(); +} + +struct mods_client *mods_client_from_id(u8 client_id) +{ + return &mp.clients[client_id - 1]; +} + +int mods_is_client_enabled(u8 client_id) +{ + return test_bit(client_id - 1, &mp.client_flags); +} + u32 mods_get_access_token(void) { return access_token; @@ -404,7 +482,9 @@ static int validate_client(struct mods_client *client) return false; } - if (client->client_id < 1 || client->client_id > MODS_MAX_CLIENTS) { + if (client->client_id < 1 || + client->client_id > MODS_MAX_CLIENTS || + !mods_is_client_enabled(client->client_id)) { cl_error("invalid client id\n"); return false; } @@ -446,6 +526,8 @@ static int __init mods_init_module(void) LOG_ENT(); + memset(&mp, 0, sizeof(mp)); + mods_init_irq(); rc = misc_register(&mods_dev); @@ -488,12 +570,10 @@ static int __init mods_init_module(void) #endif #endif -#if defined(CONFIG_ARM_FFA_TRANSPORT) -#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE) +#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) rc = mods_ffa_abi_register(); if (rc < 0) mods_warning_printk("error on mods_ffa_abi_register returned %d\n", rc); -#endif #endif mods_info_printk("*** WARNING: DIAGNOSTIC DRIVER LOADED ***\n"); @@ -510,13 +590,18 @@ static int __init mods_init_module(void) static void __exit mods_exit_module(void) { + int i; + LOG_ENT(); mods_exit_dmabuf(); mods_remove_debugfs(); - mods_cleanup_irq(); + for (i = 0; i < MODS_MAX_CLIENTS; i++) { + if (mp.client_flags & (1U << i)) + free_client(i + 1); + } #if defined(MODS_HAS_TEGRA) #if defined(CONFIG_DMA_ENGINE) @@ -536,12 +621,9 @@ static void __exit mods_exit_module(void) mods_shutdown_clock_api(); #endif -#if defined(CONFIG_ARM_FFA_TRANSPORT) -#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE) +#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) mods_ffa_abi_unregister(); #endif -#endif - mods_info_printk("driver unloaded\n"); LOG_EXT(); } @@ -615,6 +697,7 @@ static int register_mapping(struct mods_client *client, struct MODS_MEM_INFO *p_mem_info, phys_addr_t phys_addr, unsigned long virtual_address, + unsigned long mapping_offs, unsigned long mapping_length) { struct SYS_MAP_MEMORY *p_map_mem; @@ -630,6 +713,7 @@ static int register_mapping(struct mods_client *client, p_map_mem->phys_addr = phys_addr; p_map_mem->virtual_addr = virtual_address; + p_map_mem->mapping_offs = mapping_offs; p_map_mem->mapping_length = mapping_length; p_map_mem->p_mem_info = p_mem_info; @@ -697,9 +781,9 @@ static void unregister_all_mappings(struct mods_client *client) LOG_EXT(); } -static pgprot_t mods_get_prot(struct mods_client *client, - u8 mem_type, - pgprot_t prot) +static pgprot_t get_prot(struct mods_client *client, + u8 mem_type, + pgprot_t prot) { switch (mem_type) { case MODS_ALLOC_CACHED: @@ -717,17 +801,33 @@ static pgprot_t mods_get_prot(struct mods_client *client, } } -static pgprot_t get_prot_for_range(struct mods_client *client, - phys_addr_t phys_addr, - unsigned long size, - pgprot_t prot) +static int get_prot_for_range(struct mods_client *client, + phys_addr_t phys_addr, + unsigned long size, + pgprot_t *prot) { - if ((phys_addr == client->mem_type.phys_addr) && - (size == client->mem_type.size)) { + const phys_addr_t req_end = phys_addr + size; + const phys_addr_t range_begin = client->mem_type.phys_addr; + const phys_addr_t range_end = range_begin + client->mem_type.size; - return mods_get_prot(client, client->mem_type.type, prot); + /* Check overlap with set memory type range */ + if (phys_addr < range_end && req_end > range_begin) { + + /* Check if requested range is completely inside */ + if (likely(phys_addr >= range_begin && req_end <= range_end)) { + *prot = get_prot(client, client->mem_type.type, *prot); + return 0; + } + + cl_error("mmap range [0x%llx, 0x%llx] does not match set memory type range [0x%llx, 0x%llx]\n", + (unsigned long long)phys_addr, + (unsigned long long)req_end, + (unsigned long long)range_begin, + (unsigned long long)range_end); + return -EINVAL; } - return prot; + + return 0; } const char *mods_get_prot_str(u8 mem_type) @@ -751,13 +851,17 @@ static const char *get_prot_str_for_range(struct mods_client *client, phys_addr_t phys_addr, unsigned long size) { - if ((phys_addr == client->mem_type.phys_addr) && - (size == client->mem_type.size)) { + const phys_addr_t req_end = phys_addr + size; + const phys_addr_t range_begin = client->mem_type.phys_addr; + const phys_addr_t range_end = range_begin + client->mem_type.size; + /* Check overlap with set memory type range */ + if (phys_addr < range_end && req_end > range_begin) return mods_get_prot_str(client->mem_type.type); - } + return "default"; } + /******************** * PCI ERROR FUNCTIONS * ********************/ @@ -807,8 +911,9 @@ static void mods_krnl_vma_open(struct vm_area_struct *vma) LOG_ENT(); mods_debug_printk(DEBUG_MEM_DETAILED, - "open vma, virt 0x%lx, phys 0x%llx\n", + "open vma, virt 0x%lx, size 0x%lx, phys 0x%llx\n", vma->vm_start, + vma->vm_end - vma->vm_start, (unsigned long long)vma->vm_pgoff << PAGE_SHIFT); priv = vma->vm_private_data; @@ -829,16 +934,15 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma) struct mods_client *client = priv->client; struct SYS_MAP_MEMORY *p_map_mem; - if (unlikely(mutex_lock_interruptible(&client->mtx))) { - LOG_EXT(); - return; - } + mutex_lock(&client->mtx); /* we need to unregister the mapping */ p_map_mem = find_mapping(client, vma->vm_start); if (p_map_mem) unregister_mapping(client, p_map_mem); + mutex_unlock(&client->mtx); + mods_debug_printk(DEBUG_MEM_DETAILED, "closed vma, virt 0x%lx\n", vma->vm_start); @@ -847,8 +951,6 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma) kfree(priv); atomic_dec(&client->num_allocs); - - mutex_unlock(&client->mtx); } LOG_EXT(); @@ -877,9 +979,12 @@ static int mods_krnl_vma_access(struct vm_area_struct *vma, client = priv->client; cl_debug(DEBUG_MEM_DETAILED, - "access vma, virt 0x%lx, phys 0x%llx\n", + "access vma [virt 0x%lx, size 0x%lx, phys 0x%llx] at virt 0x%lx, len 0x%x\n", vma->vm_start, - (unsigned long long)vma->vm_pgoff << PAGE_SHIFT); + vma->vm_end - vma->vm_start, + (unsigned long long)vma->vm_pgoff << PAGE_SHIFT, + addr, + len); if (unlikely(mutex_lock_interruptible(&client->mtx))) { LOG_EXT(); @@ -891,12 +996,14 @@ static int mods_krnl_vma_access(struct vm_area_struct *vma, if (unlikely(!p_map_mem || addr < p_map_mem->virtual_addr || addr + len > p_map_mem->virtual_addr + p_map_mem->mapping_length)) { + cl_error("mapped range [virt 0x%lx, size 0x%x] does not match vma [virt 0x%lx, size 0x%lx]\n", + addr, len, vma->vm_start, vma->vm_end - vma->vm_start); mutex_unlock(&client->mtx); LOG_EXT(); return -ENOMEM; } - map_offs = addr - vma->vm_start; + map_offs = addr - vma->vm_start + p_map_mem->mapping_offs; if (p_map_mem->p_mem_info) { struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info; @@ -985,7 +1092,7 @@ static int mods_krnl_open(struct inode *ip, struct file *fp) LOG_ENT(); - client = mods_alloc_client(); + client = alloc_client(); if (client == NULL) { mods_error_printk("too many clients\n"); LOG_EXT(); @@ -1063,7 +1170,7 @@ static int mods_krnl_close(struct inode *ip, struct file *fp) client->work_queue = NULL; } - mods_free_client(client_id); + free_client(client_id); pr_info("mods [%d]: driver closed\n", client_id); @@ -1136,13 +1243,15 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma) mods_krnl_vma_open(vma); - if (unlikely(mutex_lock_interruptible(&client->mtx))) - err = -EINTR; - else { + err = mutex_lock_interruptible(&client->mtx); + if (likely(!err)) { err = mods_krnl_map_inner(client, vma); mutex_unlock(&client->mtx); } + if (unlikely(err)) + mods_krnl_vma_close(vma); + LOG_EXT(); return err; } @@ -1152,6 +1261,7 @@ static int map_system_mem(struct mods_client *client, struct MODS_MEM_INFO *p_mem_info) { struct scatterlist *sg = NULL; + const char *cache_str = mods_get_prot_str(p_mem_info->cache_type); const phys_addr_t req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; phys_addr_t reg_pa = 0; const unsigned long vma_size = vma->vm_end - vma->vm_start; @@ -1161,9 +1271,9 @@ static int map_system_mem(struct mods_client *client, const u32 num_chunks = get_num_chunks(p_mem_info); u32 map_chunks; u32 i = 0; - const pgprot_t prot = mods_get_prot(client, - p_mem_info->cache_type, - vma->vm_page_prot); + const pgprot_t prot = get_prot(client, + p_mem_info->cache_type, + vma->vm_page_prot); /* Find the beginning of the requested range */ for_each_sg(p_mem_info->sg, sg, num_chunks, i) { @@ -1200,20 +1310,22 @@ static int map_system_mem(struct mods_client *client, if (i == 0) { const phys_addr_t aoffs = req_pa - chunk_pa; - map_pa += aoffs; - map_size -= aoffs; - reg_pa = chunk_pa; + map_pa += aoffs; + map_size -= aoffs; + skip_size += aoffs; + reg_pa = chunk_pa; } if (map_size > size_to_map) map_size = (unsigned int)size_to_map; cl_debug(DEBUG_MEM_DETAILED, - "remap va 0x%lx pfn 0x%lx size 0x%x pages %u\n", + "remap va 0x%lx pfn 0x%lx size 0x%x pages %u %s\n", map_va, (unsigned long)(map_pa >> PAGE_SHIFT), map_size, - map_size >> PAGE_SHIFT); + map_size >> PAGE_SHIFT, + cache_str); if (remap_pfn_range(vma, map_va, @@ -1234,6 +1346,7 @@ static int map_system_mem(struct mods_client *client, p_mem_info, reg_pa, vma->vm_start, + skip_size, vma_size); return OK; @@ -1245,6 +1358,8 @@ static int map_device_mem(struct mods_client *client, /* device memory */ const phys_addr_t req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; const unsigned long vma_size = vma->vm_end - vma->vm_start; + pgprot_t prot = vma->vm_page_prot; + int err; cl_debug(DEBUG_MEM, "map dev: phys 0x%llx, virt 0x%lx, size 0x%lx, %s\n", @@ -1253,13 +1368,15 @@ static int map_device_mem(struct mods_client *client, vma_size, get_prot_str_for_range(client, req_pa, vma_size)); - if (io_remap_pfn_range( - vma, - vma->vm_start, - vma->vm_pgoff, - vma_size, - get_prot_for_range(client, req_pa, vma_size, - vma->vm_page_prot))) { + err = get_prot_for_range(client, req_pa, vma_size, &prot); + if (unlikely(err)) + return err; + + if (unlikely(io_remap_pfn_range(vma, + vma->vm_start, + vma->vm_pgoff, + vma_size, + prot))) { cl_error("failed to map device memory\n"); return -EAGAIN; } @@ -1268,6 +1385,7 @@ static int map_device_mem(struct mods_client *client, NULL, req_pa, vma->vm_start, + 0, vma_size); return OK; @@ -2664,12 +2782,10 @@ static long mods_krnl_ioctl(struct file *fp, #endif -#if defined(CONFIG_ARM_FFA_TRANSPORT) -#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE) +#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) case MODS_ESC_FFA_CMD: MODS_IOCTL(MODS_ESC_FFA_CMD, esc_mods_arm_ffa_cmd, MODS_FFA_PARAMS); break; -#endif #endif case MODS_ESC_ACQUIRE_ACCESS_TOKEN: diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c index f896705b..65530f20 100644 --- a/drivers/misc/mods/mods_mem.c +++ b/drivers/misc/mods/mods_mem.c @@ -22,6 +22,7 @@ #include #include +#include #if defined(MODS_HAS_SET_DMA_MASK) #include @@ -412,7 +413,6 @@ static int dma_map_to_default_dev(struct mods_client *client, #ifdef CONFIG_ARM64 static void clear_contiguous_cache(struct mods_client *client, u64 virt_start, - u64 phys_start, u32 size); #endif @@ -438,7 +438,6 @@ static int setup_cache_attr(struct mods_client *client, #ifdef CONFIG_ARM64 clear_contiguous_cache(client, (u64)(size_t)ptr, - sg_phys(sg) + offs, PAGE_SIZE); #else if (p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE) @@ -458,6 +457,9 @@ static int setup_cache_attr(struct mods_client *client, * even for chunks where we haven't change them. */ mark_chunk_wc(p_mem_info, ichunk); + + /* Avoid superficial lockups */ + cond_resched(); } } @@ -2077,7 +2079,6 @@ int esc_mods_virtual_to_phys(struct mods_client *client, list_for_each(iter, head) { struct SYS_MAP_MEMORY *p_map_mem; u64 begin, end; - u64 phys_offs; p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list); @@ -2104,14 +2105,9 @@ int esc_mods_virtual_to_phys(struct mods_client *client, return OK; } - if (get_alloc_offset(p_map_mem->p_mem_info, - p_map_mem->phys_addr, - &phys_offs) != OK) - break; - range.memory_handle = (u64)(size_t)p_map_mem->p_mem_info; - range.offset = virt_offs + phys_offs; + range.offset = virt_offs + p_map_mem->mapping_offs; mutex_unlock(&client->mtx); @@ -2194,10 +2190,7 @@ int esc_mods_phys_to_virtual(struct mods_client *client, continue; /* offset from the beginning of the mapping */ - if (get_alloc_offset(p_map_mem->p_mem_info, - p_map_mem->phys_addr, - &map_offset)) - continue; + map_offset = p_map_mem->mapping_offs; if ((offset >= map_offset) && (offset < map_offset + p_map_mem->mapping_length)) { @@ -2474,7 +2467,6 @@ failed: #ifdef CONFIG_ARM64 static void clear_contiguous_cache(struct mods_client *client, u64 virt_start, - u64 phys_start, u32 size) { u64 end = virt_start + size; @@ -2483,16 +2475,16 @@ static void clear_contiguous_cache(struct mods_client *client, static u32 d_line_shift; if (!d_line_shift) { -#if KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION - const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); #if KERNEL_VERSION(6, 0, 0) <= MODS_KERNEL_VERSION - const int field = CTR_EL0_DminLine_SHIFT; -#else - const int field = CTR_DMINLINE_SHIFT; -#endif + const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); d_line_shift = - cpuid_feature_extract_unsigned_field(ctr_el0, field); + cpuid_feature_extract_unsigned_field(ctr_el0, CTR_EL0_DminLine_SHIFT); +#elif KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION + const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0); + + d_line_shift = + cpuid_feature_extract_unsigned_field(ctr_el0, CTR_DMINLINE_SHIFT); #else d_line_shift = 4; /* Fallback for kernel 5.9 or older */ #endif @@ -2502,91 +2494,15 @@ static void clear_contiguous_cache(struct mods_client *client, cur = virt_start & ~(d_size - 1); do { asm volatile("dc civac, %0" : : "r" (cur) : "memory"); + + /* Avoid superficial lockups */ + cond_resched(); } while (cur += d_size, cur < end); + asm volatile("dsb sy" : : : "memory"); cl_debug(DEBUG_MEM_DETAILED, - "clear cache virt 0x%llx phys 0x%llx size 0x%x\n", - virt_start, phys_start, size); -} - -static void clear_entry_cache_mappings(struct mods_client *client, - struct SYS_MAP_MEMORY *p_map_mem, - u64 virt_offs, - u64 virt_offs_end) -{ - struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info; - struct scatterlist *sg; - u64 cur_vo = p_map_mem->virtual_addr; - u32 num_chunks; - u32 i; - - if (!p_mem_info) - return; - - if (p_mem_info->cache_type != MODS_ALLOC_CACHED) - return; - - num_chunks = get_num_chunks(p_mem_info); - - for_each_sg(p_mem_info->sg, sg, num_chunks, i) { - u32 chunk_offs = 0; - u32 chunk_offs_end = sg->length; - u64 cur_vo_end = cur_vo + chunk_offs_end; - - if (virt_offs_end <= cur_vo) - break; - - if (virt_offs >= cur_vo_end) { - cur_vo = cur_vo_end; - continue; - } - - if (cur_vo < virt_offs) - chunk_offs = (u32)(virt_offs - cur_vo); - - if (virt_offs_end < cur_vo_end) - chunk_offs_end -= (u32)(cur_vo_end - virt_offs_end); - - cl_debug(DEBUG_MEM_DETAILED, - "clear cache %p [%u]\n", - p_mem_info, - i); - - while (chunk_offs < chunk_offs_end) { - u32 i_page = chunk_offs >> PAGE_SHIFT; - u32 page_offs = chunk_offs - (i_page << PAGE_SHIFT); - u64 page_va = - (u64)(size_t)MODS_KMAP(sg_page(sg) + i_page); - u64 clear_va = page_va + page_offs; - u64 clear_pa = sg_phys(sg) + chunk_offs; - u32 clear_size = PAGE_SIZE - page_offs; - u64 remaining = chunk_offs_end - chunk_offs; - - if (likely(page_va)) { - if ((u64)clear_size > remaining) - clear_size = (u32)remaining; - - cl_debug(DEBUG_MEM_DETAILED, - "clear page %u, chunk offs 0x%x, page va 0x%llx\n", - i_page, - chunk_offs, - page_va); - - clear_contiguous_cache(client, - clear_va, - clear_pa, - clear_size); - - kunmap((void *)(size_t)page_va); - } else { - cl_error("kmap failed\n"); - } - - chunk_offs += clear_size; - } - - cur_vo = cur_vo_end; - } + "flush cache virt 0x%llx size 0x%x\n", + virt_start, size); } int esc_mods_flush_cpu_cache_range(struct mods_client *client, @@ -2619,45 +2535,34 @@ int esc_mods_flush_cpu_cache_range(struct mods_client *client, } head = &client->mem_map_list; + err = -EINVAL; list_for_each(iter, head) { struct SYS_MAP_MEMORY *p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list); - u64 mapped_va = p_map_mem->virtual_addr; + const u64 mapped_va = p_map_mem->virtual_addr; + const u64 mapped_end = mapped_va + p_map_mem->mapping_length; + const u64 flush_start = p->virt_addr_start < mapped_va ? mapped_va + : p->virt_addr_start; + const u64 flush_end = p->virt_addr_end > mapped_end ? mapped_end + : p->virt_addr_end; - /* Note: mapping end points to the first address of next range*/ - u64 mapping_end = mapped_va + p_map_mem->mapping_length; + if (flush_start >= flush_end) + continue; - int start_on_page = p->virt_addr_start >= mapped_va - && p->virt_addr_start < mapping_end; - int start_before_page = p->virt_addr_start < mapped_va; - int end_on_page = p->virt_addr_end >= mapped_va - && p->virt_addr_end < mapping_end; - int end_after_page = p->virt_addr_end >= mapping_end; - u64 virt_start = p->virt_addr_start; - - /* Kernel expects end to point to the first address of next - * range - */ - u64 virt_end = p->virt_addr_end + 1; - - if ((start_on_page || start_before_page) - && (end_on_page || end_after_page)) { - - if (!start_on_page) - virt_start = p_map_mem->virtual_addr; - if (!end_on_page) - virt_end = mapping_end; - clear_entry_cache_mappings(client, - p_map_mem, - virt_start, - virt_end); - } + clear_contiguous_cache(client, flush_start, flush_end - flush_start); + err = OK; } + mutex_unlock(&client->mtx); + if (err) + cl_error("va range 0x%lx..0x%lx not flushed\n", + (unsigned long)p->virt_addr_start, + (unsigned long)p->virt_addr_end); + LOG_EXT(); - return OK; + return err; } #endif /* CONFIG_ARM64 */ diff --git a/include/uapi/misc/mods.h b/include/uapi/misc/mods.h index 9ed935ac..d15ce23a 100644 --- a/include/uapi/misc/mods.h +++ b/include/uapi/misc/mods.h @@ -25,7 +25,7 @@ /* Driver version */ #define MODS_DRIVER_VERSION_MAJOR 4 -#define MODS_DRIVER_VERSION_MINOR 13 +#define MODS_DRIVER_VERSION_MINOR 15 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \ ((MODS_DRIVER_VERSION_MINOR / 10) << 4) | \ (MODS_DRIVER_VERSION_MINOR % 10))