misc: mods: update from Perforce

Bug 3992588

Change-Id: I04aad94aebef7a48d9afac528347fc50c219d888
Signed-off-by: Chris Dragan <kdragan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2872373
Reviewed-by: svc-mobile-coverity <svc-mobile-coverity@nvidia.com>
Reviewed-by: Carl Dong <carld@nvidia.com>
Reviewed-by: Sachin Nikam <snikam@nvidia.com>
GVS: Gerrit_Virtual_Submit <buildbot_gerritrpt@nvidia.com>

drivers/misc/mods/Makefile

@@ -3,9 +3,6 @@
 ccflags-y += -I$(srctree.nvidia)/include
 ccflags-y += -DMODS_HAS_TEGRA
 
-LINUX_VERSION := $(shell expr $(VERSION) \* 256 + $(PATCHLEVEL))
-LINUX_VERSION_6_2 := $(shell expr 6 \* 256 + 2)
-
 ifeq ($(CONFIG_TEGRA_OOT_MODULE),m)
 CONFIG_MODS := m
 endif

drivers/misc/mods/mods_bpmpipc.c

@@ -26,9 +26,8 @@
 
 #include <soc/tegra/ivc.h>
 #include <soc/tegra/bpmp.h>
-#include <linux/version.h>
 
-#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
 #include <linux/iosys-map.h>
 #endif
 
@@ -86,7 +85,7 @@ static int bpmp_ipc_send(struct mods_client *client,
 			 const void *data,
 			 size_t sz)
 {
-#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
 	int err;
 	struct iosys_map ob;
 
@@ -119,7 +118,7 @@ static int bpmp_ipc_recv(struct mods_client *client,
 			 u32 timeout_ms)
 {
 	int err;
-#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
 	struct iosys_map ib;
 #else
 	const void *frame;
@@ -128,7 +127,7 @@ static int bpmp_ipc_recv(struct mods_client *client,
 
 	end = ktime_add_ms(ktime_get(), timeout_ms);
 
-#if (KERNEL_VERSION(6, 2, 0) <= LINUX_VERSION_CODE)
+#if (KERNEL_VERSION(6, 2, 0) <= MODS_KERNEL_VERSION)
 	do {
 		err = tegra_ivc_read_get_next_frame(ivc, &ib);
 		if (!err)

drivers/misc/mods/mods_internal.h

@@ -198,6 +198,7 @@ struct SYS_MAP_MEMORY {
 
 	phys_addr_t   phys_addr;
 	unsigned long virtual_addr;
+	unsigned long mapping_offs;   /* mapped offset from the beginning of the allocation */
 	unsigned long mapping_length; /* how many bytes were mapped */
 };
 
@@ -326,9 +327,6 @@ struct mods_priv {
 
 	/* Client structures */
 	struct mods_client clients[MODS_MAX_CLIENTS];
-
-	/* Mutex for guarding interrupt logic and PCI device enablement */
-	struct mutex       mtx;
 };
 
 #ifdef MODS_HAS_POLL_T
@@ -382,13 +380,14 @@ struct mods_priv {
 /* ** MODULE WIDE FUNCTIONS						     */
 /* ************************************************************************* */
 
+/* client */
+struct mods_client *mods_client_from_id(u8 client_id);
+int mods_is_client_enabled(u8 client_id);
+
 /* irq */
 void mods_init_irq(void);
-void mods_cleanup_irq(void);
 struct mutex *mods_get_irq_mutex(void);
-struct mods_client *mods_alloc_client(void);
 void mods_free_client_interrupts(struct mods_client *client);
-void mods_free_client(u8 client_id);
 POLL_TYPE mods_irq_event_check(u8 client_id);
 
 /* mem */

drivers/misc/mods/mods_irq.c

@@ -50,11 +50,13 @@
 /*********************
  * PRIVATE FUNCTIONS *
  *********************/
-static struct mods_priv mp;
+
+/* Mutex for guarding interrupt logic and PCI device enablement */
+static struct mutex irq_mtx;
 
 struct mutex *mods_get_irq_mutex(void)
 {
-	return &mp.mtx;
+	return &irq_mtx;
 }
 
 #ifdef CONFIG_PCI
@@ -65,7 +67,7 @@ int mods_enable_device(struct mods_client   *client,
 	int                  err   = OK;
 	struct en_dev_entry *dpriv = client->enabled_devices;
 
-	WARN_ON(!mutex_is_locked(&mp.mtx));
+	WARN_ON(!mutex_is_locked(&irq_mtx));
 
 	dpriv = pci_get_drvdata(dev);
 	if (dpriv) {
@@ -124,7 +126,7 @@ void mods_disable_device(struct mods_client *client,
 {
 	struct en_dev_entry *dpriv = pci_get_drvdata(dev);
 
-	WARN_ON(!mutex_is_locked(&mp.mtx));
+	WARN_ON(!mutex_is_locked(&irq_mtx));
 
 #ifdef MODS_HAS_SRIOV
 	if (dpriv && dpriv->num_vfs)
@@ -259,14 +261,9 @@ static const char *mods_irq_type_name(u8 irq_type)
 }
 #endif
 
-static struct mods_client *client_from_id(u8 client_id)
-{
-	return &mp.clients[client_id - 1];
-}
-
 static void wake_up_client(struct dev_irq_map *t)
 {
-	struct mods_client *client = client_from_id(t->client_id);
+	struct mods_client *client = mods_client_from_id(t->client_id);
 
 	if (client)
 		wake_up_interruptible(&client->interrupt_event);
@@ -347,7 +344,7 @@ static irqreturn_t mods_irq_handle(int irq, void *data)
 		unsigned long       flags    = 0;
 		int                 recorded = false;
 		unsigned int        irq_time = get_cur_time();
-		struct mods_client *client   = client_from_id(t->client_id);
+		struct mods_client *client   = mods_client_from_id(t->client_id);
 
 		spin_lock_irqsave(&client->irq_lock, flags);
 
@@ -385,12 +382,12 @@ static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq)
 		struct dev_irq_map *t    = NULL;
 		struct dev_irq_map *next = NULL;
 
-		if (!test_bit(client_idx - 1, &mp.client_flags))
+		if (!mods_is_client_enabled(client_idx))
 			continue;
 
 		list_for_each_entry_safe(t,
 					 next,
-					 &client_from_id(client_idx)->irq_list,
+					 &mods_client_from_id(client_idx)->irq_list,
 					 list) {
 
 			if (t->apic_irq == irq) {
@@ -657,32 +654,19 @@ void mods_init_irq(void)
 {
 	LOG_ENT();
 
-	memset(&mp, 0, sizeof(mp));
+	mutex_init(&irq_mtx);
 
-	mutex_init(&mp.mtx);
-
-	LOG_EXT();
-}
-
-void mods_cleanup_irq(void)
-{
-	int i;
-
-	LOG_ENT();
-	for (i = 0; i < MODS_MAX_CLIENTS; i++) {
-		if (mp.client_flags & (1U << i))
-			mods_free_client(i + 1);
-	}
 	LOG_EXT();
 }
 
 POLL_TYPE mods_irq_event_check(u8 client_id)
 {
-	struct irq_q_info *q = &client_from_id(client_id)->irq_queue;
+	struct irq_q_info *q;
-	unsigned int pos = (1U << (client_id - 1));
 
-	if (!(mp.client_flags & pos))
+	if (!mods_is_client_enabled(client_id))
-		return POLLERR; /* irq has quit */
+		return POLLERR; /* client has quit */
+
+	q = &mods_client_from_id(client_id)->irq_queue;
 
 	if (q->head != q->tail)
 		return POLLIN; /* irq generated */
@@ -690,51 +674,6 @@ POLL_TYPE mods_irq_event_check(u8 client_id)
 	return 0;
 }
 
-struct mods_client *mods_alloc_client(void)
-{
-	unsigned int idx;
-	unsigned int max_clients = 1;
-
-	LOG_ENT();
-
-	if (mods_get_multi_instance() ||
-	    (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE))
-		max_clients = MODS_MAX_CLIENTS;
-
-	for (idx = 1; idx <= max_clients; idx++) {
-		if (!test_and_set_bit(idx - 1U, &mp.client_flags)) {
-			struct mods_client *client = client_from_id(idx);
-
-			memset(client, 0, sizeof(*client));
-			client->client_id    = idx;
-			client->access_token = MODS_ACCESS_TOKEN_NONE;
-			atomic_set(&client->last_bad_dbdf, -1);
-
-			cl_debug(DEBUG_IOCTL,
-				 "open client (bit mask 0x%lx)\n",
-				 mp.client_flags);
-
-			mutex_init(&client->mtx);
-			spin_lock_init(&client->irq_lock);
-			init_waitqueue_head(&client->interrupt_event);
-			INIT_LIST_HEAD(&client->irq_list);
-			INIT_LIST_HEAD(&client->mem_alloc_list);
-			INIT_LIST_HEAD(&client->mem_map_list);
-			INIT_LIST_HEAD(&client->free_mem_list);
-#if defined(CONFIG_PPC64)
-			INIT_LIST_HEAD(&client->ppc_tce_bypass_list);
-			INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list);
-#endif
-
-			LOG_EXT();
-			return client;
-		}
-	}
-
-	LOG_EXT();
-	return NULL;
-}
-
 static int mods_free_irqs(struct mods_client *client,
 			  struct pci_dev     *dev)
 {
@@ -746,7 +685,7 @@ static int mods_free_irqs(struct mods_client *client,
 
 	LOG_ENT();
 
-	if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
 		LOG_EXT();
 		return -EINTR;
 	}
@@ -754,7 +693,7 @@ static int mods_free_irqs(struct mods_client *client,
 	dpriv = pci_get_drvdata(dev);
 
 	if (!dpriv) {
-		mutex_unlock(&mp.mtx);
+		mutex_unlock(&irq_mtx);
 		LOG_EXT();
 		return OK;
 	}
@@ -767,7 +706,7 @@ static int mods_free_irqs(struct mods_client *client,
 			PCI_SLOT(dev->devfn),
 			PCI_FUNC(dev->devfn),
 			dpriv->client_id);
-		mutex_unlock(&mp.mtx);
+		mutex_unlock(&irq_mtx);
 		LOG_EXT();
 		return -EINVAL;
 	}
@@ -823,7 +762,7 @@ static int mods_free_irqs(struct mods_client *client,
 	cl_debug(DEBUG_ISR_DETAILED, "irqs freed\n");
 #endif
 
-	mutex_unlock(&mp.mtx);
+	mutex_unlock(&irq_mtx);
 	LOG_EXT();
 	return 0;
 }
@@ -843,21 +782,6 @@ void mods_free_client_interrupts(struct mods_client *client)
 	LOG_EXT();
 }
 
-void mods_free_client(u8 client_id)
-{
-	struct mods_client *client = client_from_id(client_id);
-
-	LOG_ENT();
-
-	memset(client, 0, sizeof(*client));
-
-	/* Indicate the client_id is free */
-	clear_bit((unsigned int)client_id - 1U, &mp.client_flags);
-
-	cl_debug(DEBUG_IOCTL, "closed client\n");
-	LOG_EXT();
-}
-
 #ifdef CONFIG_PCI
 static int mods_allocate_irqs(struct mods_client *client,
 			      struct pci_dev     *dev,
@@ -1072,7 +996,7 @@ static int mods_register_pci_irq(struct mods_client         *client,
 		return err;
 	}
 
-	if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
 		pci_dev_put(dev);
 		LOG_EXT();
 		return -EINTR;
@@ -1088,7 +1012,7 @@ static int mods_register_pci_irq(struct mods_client         *client,
 				p->dev.device,
 				p->dev.function,
 				dpriv->client_id);
-			mutex_unlock(&mp.mtx);
+			mutex_unlock(&irq_mtx);
 			pci_dev_put(dev);
 			LOG_EXT();
 			return -EBUSY;
@@ -1100,7 +1024,7 @@ static int mods_register_pci_irq(struct mods_client         *client,
 				p->dev.bus,
 				p->dev.device,
 				p->dev.function);
-			mutex_unlock(&mp.mtx);
+			mutex_unlock(&irq_mtx);
 			pci_dev_put(dev);
 			LOG_EXT();
 			return -EINVAL;
@@ -1112,7 +1036,7 @@ static int mods_register_pci_irq(struct mods_client         *client,
 	if (err) {
 		cl_error("could not allocate irqs for irq_type %d\n",
 			 irq_type);
-		mutex_unlock(&mp.mtx);
+		mutex_unlock(&irq_mtx);
 		pci_dev_put(dev);
 		LOG_EXT();
 		return err;
@@ -1137,7 +1061,7 @@ static int mods_register_pci_irq(struct mods_client         *client,
 		}
 	}
 
-	mutex_unlock(&mp.mtx);
+	mutex_unlock(&irq_mtx);
 	pci_dev_put(dev);
 	LOG_EXT();
 	return err;
@@ -1152,7 +1076,7 @@ static int mods_register_cpu_irq(struct mods_client         *client,
 
 	LOG_ENT();
 
-	if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
 		LOG_EXT();
 		return -EINTR;
 	}
@@ -1160,7 +1084,7 @@ static int mods_register_cpu_irq(struct mods_client         *client,
 	/* Determine if the interrupt is already hooked */
 	if (mods_lookup_cpu_irq(0, irq) == IRQ_FOUND) {
 		cl_error("CPU IRQ 0x%x has already been registered\n", irq);
-		mutex_unlock(&mp.mtx);
+		mutex_unlock(&irq_mtx);
 		LOG_EXT();
 		return -EINVAL;
 	}
@@ -1168,7 +1092,7 @@ static int mods_register_cpu_irq(struct mods_client         *client,
 	/* Register interrupt */
 	err = add_irq_map(client, NULL, p, irq, 0);
 
-	mutex_unlock(&mp.mtx);
+	mutex_unlock(&irq_mtx);
 	LOG_EXT();
 	return err;
 }
@@ -1208,7 +1132,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
 
 	irq = p->dev.bus;
 
-	if (unlikely(mutex_lock_interruptible(&mp.mtx))) {
+	if (unlikely(mutex_lock_interruptible(&irq_mtx))) {
 		LOG_EXT();
 		return -EINTR;
 	}
@@ -1216,7 +1140,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
 	/* Determine if the interrupt is already hooked by this client */
 	if (mods_lookup_cpu_irq(client->client_id, irq) == IRQ_NOT_FOUND) {
 		cl_error("IRQ 0x%x not hooked, can't unhook\n", irq);
-		mutex_unlock(&mp.mtx);
+		mutex_unlock(&irq_mtx);
 		LOG_EXT();
 		return -EINVAL;
 	}
@@ -1226,7 +1150,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
 		if ((irq == del->apic_irq) && (del->dev == NULL)) {
 			if (del->type != p->type) {
 				cl_error("wrong IRQ type passed\n");
-				mutex_unlock(&mp.mtx);
+				mutex_unlock(&irq_mtx);
 				LOG_EXT();
 				return -EINVAL;
 			}
@@ -1239,7 +1163,7 @@ static int mods_unregister_cpu_irq(struct mods_client *client,
 		}
 	}
 
-	mutex_unlock(&mp.mtx);
+	mutex_unlock(&irq_mtx);
 	LOG_EXT();
 	return OK;
 }

drivers/misc/mods/mods_krnl.c

@@ -29,7 +29,6 @@
 #include <linux/random.h>
 #include <linux/sched.h>
 #include <linux/uaccess.h>
-#include <linux/version.h>
 #ifdef MODS_HAS_CONSOLE_LOCK
 #   include <linux/console.h>
 #   include <linux/kd.h>
@@ -42,6 +41,10 @@
 #   include <asm/msr.h>
 #endif
 
+#ifndef IS_BUILTIN
+#   define IS_BUILTIN(c) 0
+#endif
+
 /***********************************************************************
  * mods_krnl_* functions, driver interfaces called by the Linux kernel *
  ***********************************************************************/
@@ -392,6 +395,81 @@ int mods_get_multi_instance(void)
 	return multi_instance > 0;
 }
 
+/*********************
+ * CLIENT MANAGEMENT *
+ *********************/
+static struct mods_priv mp;
+
+static struct mods_client *alloc_client(void)
+{
+	unsigned int idx;
+	unsigned int max_clients = 1;
+
+	LOG_ENT();
+
+	if (mods_get_multi_instance() ||
+	    (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE))
+		max_clients = MODS_MAX_CLIENTS;
+
+	for (idx = 1; idx <= max_clients; idx++) {
+		if (!test_and_set_bit(idx - 1U, &mp.client_flags)) {
+			struct mods_client *client = mods_client_from_id(idx);
+
+			memset(client, 0, sizeof(*client));
+			client->client_id    = idx;
+			client->access_token = MODS_ACCESS_TOKEN_NONE;
+			atomic_set(&client->last_bad_dbdf, -1);
+
+			cl_debug(DEBUG_IOCTL,
+				 "open client (bit mask 0x%lx)\n",
+				 mp.client_flags);
+
+			mutex_init(&client->mtx);
+			spin_lock_init(&client->irq_lock);
+			init_waitqueue_head(&client->interrupt_event);
+			INIT_LIST_HEAD(&client->irq_list);
+			INIT_LIST_HEAD(&client->mem_alloc_list);
+			INIT_LIST_HEAD(&client->mem_map_list);
+			INIT_LIST_HEAD(&client->free_mem_list);
+#if defined(CONFIG_PPC64)
+			INIT_LIST_HEAD(&client->ppc_tce_bypass_list);
+			INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list);
+#endif
+
+			LOG_EXT();
+			return client;
+		}
+	}
+
+	LOG_EXT();
+	return NULL;
+}
+
+static void free_client(u8 client_id)
+{
+	struct mods_client *client = mods_client_from_id(client_id);
+
+	LOG_ENT();
+
+	memset(client, 0, sizeof(*client));
+
+	/* Indicate the client_id is free */
+	clear_bit((unsigned int)client_id - 1U, &mp.client_flags);
+
+	cl_debug(DEBUG_IOCTL, "closed client\n");
+	LOG_EXT();
+}
+
+struct mods_client *mods_client_from_id(u8 client_id)
+{
+	return &mp.clients[client_id - 1];
+}
+
+int mods_is_client_enabled(u8 client_id)
+{
+	return test_bit(client_id - 1, &mp.client_flags);
+}
+
 u32 mods_get_access_token(void)
 {
 	return access_token;
@@ -404,7 +482,9 @@ static int validate_client(struct mods_client *client)
 		return false;
 	}
 
-	if (client->client_id < 1 || client->client_id > MODS_MAX_CLIENTS) {
+	if (client->client_id < 1 ||
+	    client->client_id > MODS_MAX_CLIENTS ||
+	    !mods_is_client_enabled(client->client_id)) {
 		cl_error("invalid client id\n");
 		return false;
 	}
@@ -446,6 +526,8 @@ static int __init mods_init_module(void)
 
 	LOG_ENT();
 
+	memset(&mp, 0, sizeof(mp));
+
 	mods_init_irq();
 
 	rc = misc_register(&mods_dev);
@@ -488,12 +570,10 @@ static int __init mods_init_module(void)
 #endif
 #endif
 
-#if defined(CONFIG_ARM_FFA_TRANSPORT)
+#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
-#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE)
 	rc = mods_ffa_abi_register();
 	if (rc < 0)
 		mods_warning_printk("error on mods_ffa_abi_register returned %d\n", rc);
-#endif
 #endif
 
 	mods_info_printk("*** WARNING: DIAGNOSTIC DRIVER LOADED ***\n");
@@ -510,13 +590,18 @@ static int __init mods_init_module(void)
 
 static void __exit mods_exit_module(void)
 {
+	int i;
+
 	LOG_ENT();
 
 	mods_exit_dmabuf();
 
 	mods_remove_debugfs();
 
-	mods_cleanup_irq();
+	for (i = 0; i < MODS_MAX_CLIENTS; i++) {
+		if (mp.client_flags & (1U << i))
+			free_client(i + 1);
+	}
 
 #if defined(MODS_HAS_TEGRA)
 #if defined(CONFIG_DMA_ENGINE)
@@ -536,12 +621,9 @@ static void __exit mods_exit_module(void)
 	mods_shutdown_clock_api();
 #endif
 
-#if defined(CONFIG_ARM_FFA_TRANSPORT)
+#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
-#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE)
 	mods_ffa_abi_unregister();
 #endif
-#endif
-
 	mods_info_printk("driver unloaded\n");
 	LOG_EXT();
 }
@@ -615,6 +697,7 @@ static int register_mapping(struct mods_client   *client,
 			    struct MODS_MEM_INFO *p_mem_info,
 			    phys_addr_t           phys_addr,
 			    unsigned long         virtual_address,
+			    unsigned long         mapping_offs,
 			    unsigned long         mapping_length)
 {
 	struct SYS_MAP_MEMORY *p_map_mem;
@@ -630,6 +713,7 @@ static int register_mapping(struct mods_client   *client,
 
 	p_map_mem->phys_addr      = phys_addr;
 	p_map_mem->virtual_addr   = virtual_address;
+	p_map_mem->mapping_offs   = mapping_offs;
 	p_map_mem->mapping_length = mapping_length;
 	p_map_mem->p_mem_info     = p_mem_info;
 
@@ -697,9 +781,9 @@ static void unregister_all_mappings(struct mods_client *client)
 	LOG_EXT();
 }
 
-static pgprot_t mods_get_prot(struct mods_client *client,
+static pgprot_t get_prot(struct mods_client *client,
-			      u8                  mem_type,
+			 u8                  mem_type,
-			      pgprot_t            prot)
+			 pgprot_t            prot)
 {
 	switch (mem_type) {
 	case MODS_ALLOC_CACHED:
@@ -717,17 +801,33 @@ static pgprot_t mods_get_prot(struct mods_client *client,
 	}
 }
 
-static pgprot_t get_prot_for_range(struct mods_client *client,
+static int get_prot_for_range(struct mods_client *client,
-				   phys_addr_t         phys_addr,
+			      phys_addr_t         phys_addr,
-				   unsigned long       size,
+			      unsigned long       size,
-				   pgprot_t            prot)
+			      pgprot_t           *prot)
 {
-	if ((phys_addr == client->mem_type.phys_addr) &&
+	const phys_addr_t req_end     = phys_addr + size;
-		(size == client->mem_type.size)) {
+	const phys_addr_t range_begin = client->mem_type.phys_addr;
+	const phys_addr_t range_end   = range_begin + client->mem_type.size;
 
-		return mods_get_prot(client, client->mem_type.type, prot);
+	/* Check overlap with set memory type range */
+	if (phys_addr < range_end && req_end > range_begin) {
+
+		/* Check if requested range is completely inside */
+		if (likely(phys_addr >= range_begin && req_end <= range_end)) {
+			*prot = get_prot(client, client->mem_type.type, *prot);
+			return 0;
+		}
+
+		cl_error("mmap range [0x%llx, 0x%llx] does not match set memory type range [0x%llx, 0x%llx]\n",
+			 (unsigned long long)phys_addr,
+			 (unsigned long long)req_end,
+			 (unsigned long long)range_begin,
+			 (unsigned long long)range_end);
+		return -EINVAL;
 	}
-	return prot;
+
+	return 0;
 }
 
 const char *mods_get_prot_str(u8 mem_type)
@@ -751,13 +851,17 @@ static const char *get_prot_str_for_range(struct mods_client *client,
 					  phys_addr_t         phys_addr,
 					  unsigned long       size)
 {
-	if ((phys_addr == client->mem_type.phys_addr) &&
+	const phys_addr_t req_end     = phys_addr + size;
-		(size == client->mem_type.size)) {
+	const phys_addr_t range_begin = client->mem_type.phys_addr;
+	const phys_addr_t range_end   = range_begin + client->mem_type.size;
 
+	/* Check overlap with set memory type range */
+	if (phys_addr < range_end && req_end > range_begin)
 		return mods_get_prot_str(client->mem_type.type);
-	}
+
 	return "default";
 }
+
 /********************
  * PCI ERROR FUNCTIONS *
  ********************/
@@ -807,8 +911,9 @@ static void mods_krnl_vma_open(struct vm_area_struct *vma)
 
 	LOG_ENT();
 	mods_debug_printk(DEBUG_MEM_DETAILED,
-			  "open vma, virt 0x%lx, phys 0x%llx\n",
+			  "open vma, virt 0x%lx, size 0x%lx, phys 0x%llx\n",
 			  vma->vm_start,
+			  vma->vm_end - vma->vm_start,
 			  (unsigned long long)vma->vm_pgoff << PAGE_SHIFT);
 
 	priv = vma->vm_private_data;
@@ -829,16 +934,15 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma)
 		struct mods_client    *client = priv->client;
 		struct SYS_MAP_MEMORY *p_map_mem;
 
-		if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+		mutex_lock(&client->mtx);
-			LOG_EXT();
-			return;
-		}
 
 		/* we need to unregister the mapping */
 		p_map_mem = find_mapping(client, vma->vm_start);
 		if (p_map_mem)
 			unregister_mapping(client, p_map_mem);
 
+		mutex_unlock(&client->mtx);
+
 		mods_debug_printk(DEBUG_MEM_DETAILED,
 				  "closed vma, virt 0x%lx\n",
 				  vma->vm_start);
@@ -847,8 +951,6 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma)
 
 		kfree(priv);
 		atomic_dec(&client->num_allocs);
-
-		mutex_unlock(&client->mtx);
 	}
 
 	LOG_EXT();
@@ -877,9 +979,12 @@ static int mods_krnl_vma_access(struct vm_area_struct *vma,
 	client = priv->client;
 
 	cl_debug(DEBUG_MEM_DETAILED,
-		 "access vma, virt 0x%lx, phys 0x%llx\n",
+		 "access vma [virt 0x%lx, size 0x%lx, phys 0x%llx] at virt 0x%lx, len 0x%x\n",
 		 vma->vm_start,
-		 (unsigned long long)vma->vm_pgoff << PAGE_SHIFT);
+		 vma->vm_end - vma->vm_start,
+		 (unsigned long long)vma->vm_pgoff << PAGE_SHIFT,
+		 addr,
+		 len);
 
 	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
 		LOG_EXT();
@@ -891,12 +996,14 @@ static int mods_krnl_vma_access(struct vm_area_struct *vma,
 	if (unlikely(!p_map_mem || addr < p_map_mem->virtual_addr ||
 		     addr + len > p_map_mem->virtual_addr +
 				  p_map_mem->mapping_length)) {
+		cl_error("mapped range [virt 0x%lx, size 0x%x] does not match vma [virt 0x%lx, size 0x%lx]\n",
+			 addr, len, vma->vm_start, vma->vm_end - vma->vm_start);
 		mutex_unlock(&client->mtx);
 		LOG_EXT();
 		return -ENOMEM;
 	}
 
-	map_offs = addr - vma->vm_start;
+	map_offs = addr - vma->vm_start + p_map_mem->mapping_offs;
 
 	if (p_map_mem->p_mem_info) {
 		struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info;
@@ -985,7 +1092,7 @@ static int mods_krnl_open(struct inode *ip, struct file *fp)
 
 	LOG_ENT();
 
-	client = mods_alloc_client();
+	client = alloc_client();
 	if (client == NULL) {
 		mods_error_printk("too many clients\n");
 		LOG_EXT();
@@ -1063,7 +1170,7 @@ static int mods_krnl_close(struct inode *ip, struct file *fp)
 		client->work_queue = NULL;
 	}
 
-	mods_free_client(client_id);
+	free_client(client_id);
 
 	pr_info("mods [%d]: driver closed\n", client_id);
 
@@ -1136,13 +1243,15 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma)
 
 	mods_krnl_vma_open(vma);
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+	err = mutex_lock_interruptible(&client->mtx);
-		err = -EINTR;
+	if (likely(!err)) {
-	else {
 		err = mods_krnl_map_inner(client, vma);
 		mutex_unlock(&client->mtx);
 	}
 
+	if (unlikely(err))
+		mods_krnl_vma_close(vma);
+
 	LOG_EXT();
 	return err;
 }
@@ -1152,6 +1261,7 @@ static int map_system_mem(struct mods_client    *client,
 			  struct MODS_MEM_INFO  *p_mem_info)
 {
 	struct scatterlist *sg          = NULL;
+	const char         *cache_str   = mods_get_prot_str(p_mem_info->cache_type);
 	const phys_addr_t   req_pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
 	phys_addr_t         reg_pa      = 0;
 	const unsigned long vma_size    = vma->vm_end - vma->vm_start;
@@ -1161,9 +1271,9 @@ static int map_system_mem(struct mods_client    *client,
 	const u32           num_chunks  = get_num_chunks(p_mem_info);
 	u32                 map_chunks;
 	u32                 i           = 0;
-	const pgprot_t      prot        = mods_get_prot(client,
+	const pgprot_t      prot        = get_prot(client,
-							p_mem_info->cache_type,
+						   p_mem_info->cache_type,
-							vma->vm_page_prot);
+						   vma->vm_page_prot);
 
 	/* Find the beginning of the requested range */
 	for_each_sg(p_mem_info->sg, sg, num_chunks, i) {
@@ -1200,20 +1310,22 @@ static int map_system_mem(struct mods_client    *client,
 		if (i == 0) {
 			const phys_addr_t aoffs = req_pa - chunk_pa;
 
-			map_pa   += aoffs;
+			map_pa    += aoffs;
-			map_size -= aoffs;
+			map_size  -= aoffs;
-			reg_pa   =  chunk_pa;
+			skip_size += aoffs;
+			reg_pa     = chunk_pa;
 		}
 
 		if (map_size > size_to_map)
 			map_size = (unsigned int)size_to_map;
 
 		cl_debug(DEBUG_MEM_DETAILED,
-			 "remap va 0x%lx pfn 0x%lx size 0x%x pages %u\n",
+			 "remap va 0x%lx pfn 0x%lx size 0x%x pages %u %s\n",
 			 map_va,
 			 (unsigned long)(map_pa >> PAGE_SHIFT),
 			 map_size,
-			 map_size >> PAGE_SHIFT);
+			 map_size >> PAGE_SHIFT,
+			 cache_str);
 
 		if (remap_pfn_range(vma,
 				    map_va,
@@ -1234,6 +1346,7 @@ static int map_system_mem(struct mods_client    *client,
 			 p_mem_info,
 			 reg_pa,
 			 vma->vm_start,
+			 skip_size,
 			 vma_size);
 
 	return OK;
@@ -1245,6 +1358,8 @@ static int map_device_mem(struct mods_client    *client,
 	/* device memory */
 	const phys_addr_t   req_pa   = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT;
 	const unsigned long vma_size = vma->vm_end - vma->vm_start;
+	pgprot_t            prot     = vma->vm_page_prot;
+	int                 err;
 
 	cl_debug(DEBUG_MEM,
 		 "map dev: phys 0x%llx, virt 0x%lx, size 0x%lx, %s\n",
@@ -1253,13 +1368,15 @@ static int map_device_mem(struct mods_client    *client,
 		 vma_size,
 		 get_prot_str_for_range(client, req_pa, vma_size));
 
-	if (io_remap_pfn_range(
+	err = get_prot_for_range(client, req_pa, vma_size, &prot);
-			vma,
+	if (unlikely(err))
-			vma->vm_start,
+		return err;
-			vma->vm_pgoff,
+
-			vma_size,
+	if (unlikely(io_remap_pfn_range(vma,
-			get_prot_for_range(client, req_pa, vma_size,
+					vma->vm_start,
-					   vma->vm_page_prot))) {
+					vma->vm_pgoff,
+					vma_size,
+					prot))) {
 		cl_error("failed to map device memory\n");
 		return -EAGAIN;
 	}
@@ -1268,6 +1385,7 @@ static int map_device_mem(struct mods_client    *client,
 			 NULL,
 			 req_pa,
 			 vma->vm_start,
+			 0,
 			 vma_size);
 
 	return OK;
@@ -2664,12 +2782,10 @@ static long mods_krnl_ioctl(struct file  *fp,
 
 #endif
 
-#if defined(CONFIG_ARM_FFA_TRANSPORT)
+#if IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT)
-#if (KERNEL_VERSION(6, 2, 0) > LINUX_VERSION_CODE)
 	case MODS_ESC_FFA_CMD:
 		MODS_IOCTL(MODS_ESC_FFA_CMD, esc_mods_arm_ffa_cmd, MODS_FFA_PARAMS);
 		break;
-#endif
 #endif
 
 	case MODS_ESC_ACQUIRE_ACCESS_TOKEN:

drivers/misc/mods/mods_mem.c

@@ -22,6 +22,7 @@
 
 #include <linux/bitops.h>
 #include <linux/pagemap.h>
+#include <linux/sched.h>
 
 #if defined(MODS_HAS_SET_DMA_MASK)
 #include <linux/dma-mapping.h>
@@ -412,7 +413,6 @@ static int dma_map_to_default_dev(struct mods_client   *client,
 #ifdef CONFIG_ARM64
 static void clear_contiguous_cache(struct mods_client *client,
 				   u64                 virt_start,
-				   u64                 phys_start,
 				   u32                 size);
 #endif
 
@@ -438,7 +438,6 @@ static int setup_cache_attr(struct mods_client   *client,
 #ifdef CONFIG_ARM64
 			clear_contiguous_cache(client,
 					       (u64)(size_t)ptr,
-					       sg_phys(sg) + offs,
 					       PAGE_SIZE);
 #else
 			if (p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE)
@@ -458,6 +457,9 @@ static int setup_cache_attr(struct mods_client   *client,
 			 * even for chunks where we haven't change them.
 			 */
 			mark_chunk_wc(p_mem_info, ichunk);
+
+			/* Avoid superficial lockups */
+			cond_resched();
 		}
 	}
 
@@ -2077,7 +2079,6 @@ int esc_mods_virtual_to_phys(struct mods_client              *client,
 	list_for_each(iter, head) {
 		struct SYS_MAP_MEMORY *p_map_mem;
 		u64                    begin, end;
-		u64                    phys_offs;
 
 		p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list);
 
@@ -2104,14 +2105,9 @@ int esc_mods_virtual_to_phys(struct mods_client              *client,
 				return OK;
 			}
 
-			if (get_alloc_offset(p_map_mem->p_mem_info,
-					     p_map_mem->phys_addr,
-					     &phys_offs) != OK)
-				break;
-
 			range.memory_handle =
 				(u64)(size_t)p_map_mem->p_mem_info;
-			range.offset = virt_offs + phys_offs;
+			range.offset = virt_offs + p_map_mem->mapping_offs;
 
 			mutex_unlock(&client->mtx);
 
@@ -2194,10 +2190,7 @@ int esc_mods_phys_to_virtual(struct mods_client              *client,
 			continue;
 
 		/* offset from the beginning of the mapping */
-		if (get_alloc_offset(p_map_mem->p_mem_info,
+		map_offset = p_map_mem->mapping_offs;
-				     p_map_mem->phys_addr,
-				     &map_offset))
-			continue;
 
 		if ((offset >= map_offset) &&
 		    (offset <  map_offset + p_map_mem->mapping_length)) {
@@ -2474,7 +2467,6 @@ failed:
 #ifdef CONFIG_ARM64
 static void clear_contiguous_cache(struct mods_client *client,
 				   u64                 virt_start,
-				   u64                 phys_start,
 				   u32                 size)
 {
 	u64 end = virt_start + size;
@@ -2483,16 +2475,16 @@ static void clear_contiguous_cache(struct mods_client *client,
 	static u32 d_line_shift;
 
 	if (!d_line_shift) {
-#if KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION
-		const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
 #if KERNEL_VERSION(6, 0, 0) <= MODS_KERNEL_VERSION
-		const int field = CTR_EL0_DminLine_SHIFT;
+		const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
-#else
-		const int field = CTR_DMINLINE_SHIFT;
-#endif
 
 		d_line_shift =
-			cpuid_feature_extract_unsigned_field(ctr_el0, field);
+			cpuid_feature_extract_unsigned_field(ctr_el0, CTR_EL0_DminLine_SHIFT);
+#elif KERNEL_VERSION(5, 10, 0) <= MODS_KERNEL_VERSION
+		const u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
+
+		d_line_shift =
+			cpuid_feature_extract_unsigned_field(ctr_el0, CTR_DMINLINE_SHIFT);
 #else
 		d_line_shift = 4; /* Fallback for kernel 5.9 or older */
 #endif
@@ -2502,91 +2494,15 @@ static void clear_contiguous_cache(struct mods_client *client,
 	cur = virt_start & ~(d_size - 1);
 	do {
 		asm volatile("dc civac, %0" : : "r" (cur) : "memory");
+
+		/* Avoid superficial lockups */
+		cond_resched();
 	} while (cur += d_size, cur < end);
+	asm volatile("dsb sy" : : : "memory");
 
 	cl_debug(DEBUG_MEM_DETAILED,
-		 "clear cache virt 0x%llx phys 0x%llx size 0x%x\n",
+		 "flush cache virt 0x%llx size 0x%x\n",
-		 virt_start, phys_start, size);
+		 virt_start, size);
-}
-
-static void clear_entry_cache_mappings(struct mods_client    *client,
-				       struct SYS_MAP_MEMORY *p_map_mem,
-				       u64                    virt_offs,
-				       u64                    virt_offs_end)
-{
-	struct MODS_MEM_INFO *p_mem_info = p_map_mem->p_mem_info;
-	struct scatterlist   *sg;
-	u64	              cur_vo = p_map_mem->virtual_addr;
-	u32                   num_chunks;
-	u32                   i;
-
-	if (!p_mem_info)
-		return;
-
-	if (p_mem_info->cache_type != MODS_ALLOC_CACHED)
-		return;
-
-	num_chunks = get_num_chunks(p_mem_info);
-
-	for_each_sg(p_mem_info->sg, sg, num_chunks, i) {
-		u32 chunk_offs     = 0;
-		u32 chunk_offs_end = sg->length;
-		u64 cur_vo_end     = cur_vo + chunk_offs_end;
-
-		if (virt_offs_end <= cur_vo)
-			break;
-
-		if (virt_offs >= cur_vo_end) {
-			cur_vo = cur_vo_end;
-			continue;
-		}
-
-		if (cur_vo < virt_offs)
-			chunk_offs = (u32)(virt_offs - cur_vo);
-
-		if (virt_offs_end < cur_vo_end)
-			chunk_offs_end -= (u32)(cur_vo_end - virt_offs_end);
-
-		cl_debug(DEBUG_MEM_DETAILED,
-			 "clear cache %p [%u]\n",
-			 p_mem_info,
-			 i);
-
-		while (chunk_offs < chunk_offs_end) {
-			u32 i_page     = chunk_offs >> PAGE_SHIFT;
-			u32 page_offs  = chunk_offs - (i_page << PAGE_SHIFT);
-			u64 page_va    =
-			    (u64)(size_t)MODS_KMAP(sg_page(sg) + i_page);
-			u64 clear_va   = page_va + page_offs;
-			u64 clear_pa   = sg_phys(sg) + chunk_offs;
-			u32 clear_size = PAGE_SIZE - page_offs;
-			u64 remaining  = chunk_offs_end - chunk_offs;
-
-			if (likely(page_va)) {
-				if ((u64)clear_size > remaining)
-					clear_size = (u32)remaining;
-
-				cl_debug(DEBUG_MEM_DETAILED,
-					 "clear page %u, chunk offs 0x%x, page va 0x%llx\n",
-					 i_page,
-					 chunk_offs,
-					 page_va);
-
-				clear_contiguous_cache(client,
-						       clear_va,
-						       clear_pa,
-						       clear_size);
-
-				kunmap((void *)(size_t)page_va);
-			} else {
-				cl_error("kmap failed\n");
-			}
-
-			chunk_offs += clear_size;
-		}
-
-		cur_vo = cur_vo_end;
-	}
 }
 
 int esc_mods_flush_cpu_cache_range(struct mods_client                *client,
@@ -2619,45 +2535,34 @@ int esc_mods_flush_cpu_cache_range(struct mods_client                *client,
 	}
 
 	head = &client->mem_map_list;
+	err  = -EINVAL;
 
 	list_for_each(iter, head) {
 		struct SYS_MAP_MEMORY *p_map_mem
 			= list_entry(iter, struct SYS_MAP_MEMORY, list);
 
-		u64 mapped_va = p_map_mem->virtual_addr;
+		const u64 mapped_va   = p_map_mem->virtual_addr;
+		const u64 mapped_end  = mapped_va + p_map_mem->mapping_length;
+		const u64 flush_start = p->virt_addr_start < mapped_va ? mapped_va
+								       : p->virt_addr_start;
+		const u64 flush_end   = p->virt_addr_end > mapped_end ? mapped_end
+								      : p->virt_addr_end;
 
-		/* Note: mapping end points to the first address of next range*/
+		if (flush_start >= flush_end)
-		u64 mapping_end = mapped_va + p_map_mem->mapping_length;
+			continue;
 
-		int start_on_page = p->virt_addr_start >= mapped_va
+		clear_contiguous_cache(client, flush_start, flush_end - flush_start);
-				    && p->virt_addr_start < mapping_end;
+		err = OK;
-		int start_before_page = p->virt_addr_start < mapped_va;
-		int end_on_page = p->virt_addr_end >= mapped_va
-				  && p->virt_addr_end < mapping_end;
-		int end_after_page = p->virt_addr_end >= mapping_end;
-		u64 virt_start = p->virt_addr_start;
-
-		/* Kernel expects end to point to the first address of next
-		 * range
-		 */
-		u64 virt_end = p->virt_addr_end + 1;
-
-		if ((start_on_page || start_before_page)
-			&& (end_on_page || end_after_page)) {
-
-			if (!start_on_page)
-				virt_start = p_map_mem->virtual_addr;
-			if (!end_on_page)
-				virt_end = mapping_end;
-			clear_entry_cache_mappings(client,
-						   p_map_mem,
-						   virt_start,
-						   virt_end);
-		}
 	}
+
 	mutex_unlock(&client->mtx);
 
+	if (err)
+		cl_error("va range 0x%lx..0x%lx not flushed\n",
+			 (unsigned long)p->virt_addr_start,
+			 (unsigned long)p->virt_addr_end);
+
 	LOG_EXT();
-	return OK;
+	return err;
 }
 #endif /* CONFIG_ARM64 */

include/uapi/misc/mods.h

@@ -25,7 +25,7 @@
 
 /* Driver version */
 #define MODS_DRIVER_VERSION_MAJOR 4
-#define MODS_DRIVER_VERSION_MINOR 13
+#define MODS_DRIVER_VERSION_MINOR 15
 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \
 			     ((MODS_DRIVER_VERSION_MINOR / 10) << 4) | \
 			     (MODS_DRIVER_VERSION_MINOR % 10))