misc: mods: update MODS kernel driver to 3.99

Change-Id: I7ca22718af4e4f897ec0d410949fa1c14022eec1
Signed-off-by: Chris Dragan <kdragan@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2422786
Reviewed-by: automaticguardword <automaticguardword@nvidia.com>
Reviewed-by: Automatic_Commit_Validation_User
Reviewed-by: Sachin Nikam <snikam@nvidia.com>
Reviewed-by: Bharat Nihalani <bnihalani@nvidia.com>
Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com>
GVS: Gerrit_Virtual_Submit
Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>

drivers/misc/mods/mods_acpi.c

@@ -151,6 +151,17 @@ static int mods_extract_acpi_object(struct mods_client *client,
 		}
 		break;
 
+	case ACPI_TYPE_LOCAL_REFERENCE:
+		if (obj->reference.actual_type == ACPI_TYPE_POWER) {
+			memcpy(*buf, &obj->reference.handle,
+					sizeof(obj->reference.handle));
+			*buf += sizeof(obj->reference.handle);
+		} else {
+			cl_error("Unsupported ACPI reference type\n");
+			err = -EINVAL;
+		}
+		break;
+
 	default:
 		cl_error("unsupported ACPI output type 0x%02x from method %s\n",
 			 (unsigned int)obj->type,
@@ -269,6 +280,9 @@ static int mods_eval_acpi_method(struct mods_client           *client,
 		return -EINVAL;
 	}
 
+	input.count = p->argument_count;
+	input.pointer = acpi_params;
+
 	for (i = 0; i < p->argument_count; i++) {
 		switch (p->argument[i].type) {
 		case ACPI_MODS_TYPE_INTEGER: {
@@ -285,6 +299,28 @@ static int mods_eval_acpi_method(struct mods_client           *client,
 				= p->in_buffer + p->argument[i].buffer.offset;
 			break;
 		}
+		case ACPI_MODS_TYPE_METHOD: {
+			memcpy(&acpi_method_handler,
+					&p->argument[i].method.handle,
+					sizeof(acpi_method_handler));
+
+			if (!acpi_method_handler) {
+				cl_error("ACPI: Invalid reference handle 0\n");
+				pci_dev_put(dev);
+				LOG_EXT();
+				return -EINVAL;
+			}
+
+			if (i != p->argument_count - 1) {
+				cl_error("ACPI: Invalid argument count\n");
+				pci_dev_put(dev);
+				LOG_EXT();
+				return -EINVAL;
+			}
+
+			--input.count;
+			break;
+		}
 		default: {
 			cl_error("unsupported ACPI argument type\n");
 			pci_dev_put(dev);
@@ -294,9 +330,6 @@ static int mods_eval_acpi_method(struct mods_client           *client,
 		}
 	}
 
-	input.count   = p->argument_count;
-	input.pointer = acpi_params;
-
 	status = acpi_evaluate_object(acpi_method_handler,
 				      pdevice ? p->method_name : NULL,
 				      &input,

drivers/misc/mods/mods_internal.h

@@ -543,6 +543,8 @@ int esc_mods_get_iommu_state_2(struct mods_client          *client,
 			       struct MODS_GET_IOMMU_STATE *state);
 int esc_mods_pci_set_dma_mask(struct mods_client      *client,
 			     struct MODS_PCI_DMA_MASK *dma_mask);
+int esc_mods_pci_reset_function(struct mods_client    *client,
+				struct mods_pci_dev_2 *pcidev);
 #endif
 /* irq */
 #if defined(CONFIG_ARCH_TEGRA) && defined(CONFIG_OF) && defined(CONFIG_OF_IRQ)

drivers/misc/mods/mods_krnl.c

@@ -446,8 +446,8 @@ static int __init mods_init_module(void)
 		return -EBUSY;
 #endif
 
-#if defined(CONFIG_COMMON_CLK) && defined(CONFIG_OF_RESOLVE) && \
+#if defined(CONFIG_ARCH_TEGRA) && defined(CONFIG_COMMON_CLK) && \
-		defined(CONFIG_OF_DYNAMIC)
+	defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
 	mods_init_clock_api();
 #endif
 
@@ -500,8 +500,8 @@ static void __exit mods_exit_module(void)
 
 	misc_deregister(&mods_dev);
 
-#if defined(CONFIG_COMMON_CLK) && defined(CONFIG_OF_RESOLVE) && \
+#if defined(CONFIG_ARCH_TEGRA) && defined(CONFIG_COMMON_CLK) && \
-		defined(CONFIG_OF_DYNAMIC)
+	defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
 	mods_shutdown_clock_api();
 #endif
 
@@ -1525,16 +1525,18 @@ static int esc_mods_verify_access_token(struct mods_client       *client,
 	return err;
 }
 
-struct mods_sysfs_work {
+struct mods_file_work {
 	struct work_struct work;
-	struct MODS_SYSFS_NODE *pdata;
+	const char        *path;
+	const char        *data;
+	__u32              data_size;
 	int                err;
 };
 
 static void sysfs_write_task(struct work_struct *w)
 {
-	struct mods_sysfs_work *task = container_of(w,
+	struct mods_file_work *task = container_of(w,
-						    struct mods_sysfs_work,
+						   struct mods_file_work,
 						   work);
 	struct file *f;
 	mm_segment_t old_fs;
@@ -1546,15 +1548,14 @@ static void sysfs_write_task(struct work_struct *w)
 	old_fs = get_fs();
 	set_fs(KERNEL_DS);
 
-	f = filp_open(task->pdata->path, O_WRONLY, 0);
+	f = filp_open(task->path, O_WRONLY, 0);
 	if (IS_ERR(f))
 		task->err = PTR_ERR(f);
 	else {
 		f->f_pos = 0;
-		if (task->pdata->size <= MODS_MAX_SYSFS_FILE_SIZE)
 		task->err = f->f_op->write(f,
-						   task->pdata->contents,
+					   task->data,
-						   task->pdata->size,
+					   task->data_size,
 					   &f->f_pos);
 		filp_close(f, NULL);
 	}
@@ -1564,35 +1565,82 @@ static void sysfs_write_task(struct work_struct *w)
 	LOG_EXT();
 }
 
+static int run_write_task(struct mods_client    *client,
+			  struct mods_file_work *task)
+{
+	struct workqueue_struct *wq;
+
+	wq = create_singlethread_workqueue("mods_file_write");
+	if (!wq) {
+		cl_error("failed to create work queue\n");
+		return -ENOMEM;
+	}
+
+	cl_info("write %.*s to %s\n", task->data_size, task->data, task->path);
+
+	INIT_WORK(&task->work, sysfs_write_task);
+	queue_work(wq, &task->work);
+	flush_workqueue(wq);
+	destroy_workqueue(wq);
+
+	if (task->err < 0)
+		cl_error("failed to write %.*s to %s\n",
+			 task->data_size, task->data, task->path);
+
+	return (task->err > 0) ? 0 : task->err;
+}
+
 static int esc_mods_write_sysfs_node(struct mods_client     *client,
 				     struct MODS_SYSFS_NODE *pdata)
 {
-	int err = -EINVAL;
+	int                   err;
-	struct mods_sysfs_work task;
+	struct mods_file_work task;
-	struct workqueue_struct *wq;
 
 	LOG_ENT();
 
-	memmove(&pdata->path[5], pdata->path, MODS_MAX_SYSFS_PATH_LEN);
+	if (pdata->size > MODS_MAX_SYSFS_FILE_SIZE) {
+		cl_error("invalid data size %u, max allowed is %u\n",
+			 pdata->size, MODS_MAX_SYSFS_FILE_SIZE);
+		LOG_EXT();
+		return -EINVAL;
+	}
+
+	memmove(&pdata->path[5], pdata->path, sizeof(pdata->path) - 5);
 	memcpy(pdata->path, "/sys/", 5);
-	pdata->path[MODS_MAX_SYSFS_PATH_BUF_SIZE - 1] = 0;
+	pdata->path[sizeof(pdata->path) - 1] = 0;
 
-	task.pdata = pdata;
+	task.path      = pdata->path;
+	task.data      = pdata->contents;
+	task.data_size = pdata->size;
+
+	err = run_write_task(client, &task);
 
-	wq = create_singlethread_workqueue("mods_sysfs_write");
-	if (!wq) {
 	LOG_EXT();
 	return err;
 }
 
-	INIT_WORK(&task.work, sysfs_write_task);
+static int esc_mods_sysctl_write_int(struct mods_client     *client,
-	queue_work(wq, &task.work);
+				     struct MODS_SYSCTL_INT *pdata)
-	flush_workqueue(wq);
+{
-	destroy_workqueue(wq);
+	int                   err;
+	struct mods_file_work task;
+	char                  data[21];
+	int                   data_size;
 
-	err = task.err;
+	LOG_ENT();
-	if (err > 0)
+
-		err = OK;
+	memmove(&pdata->path[10], pdata->path, sizeof(pdata->path)  - 10);
+	memcpy(pdata->path, "/proc/sys/", 10);
+	pdata->path[sizeof(pdata->path) - 1] = 0;
+
+	data_size = snprintf(data, sizeof(data),
+			     "%lld", (long long)pdata->value);
+
+	task.path      = pdata->path;
+	task.data      = data;
+	task.data_size = data_size;
+
+	err = run_write_task(client, &task);
 
 	LOG_EXT();
 	return err;
@@ -1862,6 +1910,12 @@ static long mods_krnl_ioctl(struct file  *fp,
 			   esc_mods_pci_set_dma_mask,
 			   MODS_PCI_DMA_MASK);
 		break;
+
+	case MODS_ESC_PCI_RESET_FUNCTION:
+		MODS_IOCTL(MODS_ESC_PCI_RESET_FUNCTION,
+			   esc_mods_pci_reset_function,
+			   mods_pci_dev_2);
+		break;
 #endif
 
 	case MODS_ESC_ALLOC_PAGES:
@@ -2114,8 +2168,8 @@ static long mods_krnl_ioctl(struct file  *fp,
 			   esc_mods_get_kernel_version, MODS_GET_VERSION);
 		break;
 
-#if defined(CONFIG_COMMON_CLK) && defined(CONFIG_OF_RESOLVE) && \
+#if defined(CONFIG_ARCH_TEGRA) && defined(CONFIG_COMMON_CLK) && \
-		defined(CONFIG_OF_DYNAMIC)
+	defined(CONFIG_OF_RESOLVE) && defined(CONFIG_OF_DYNAMIC)
 	case MODS_ESC_GET_CLOCK_HANDLE:
 		MODS_IOCTL(MODS_ESC_GET_CLOCK_HANDLE,
 			   esc_mods_get_clock_handle, MODS_GET_CLOCK_HANDLE);
@@ -2398,6 +2452,12 @@ static long mods_krnl_ioctl(struct file  *fp,
 				    MODS_SYSFS_NODE);
 		break;
 
+	case MODS_ESC_SYSCTL_WRITE_INT:
+		MODS_IOCTL_NORETVAL(MODS_ESC_SYSCTL_WRITE_INT,
+				    esc_mods_sysctl_write_int,
+				    MODS_SYSCTL_INT);
+		break;
+
 	case MODS_ESC_REGISTER_IRQ_4:
 		MODS_IOCTL_NORETVAL(MODS_ESC_REGISTER_IRQ_4,
 				esc_mods_register_irq_4, MODS_REGISTER_IRQ_4);

drivers/misc/mods/mods_mem.c

@@ -290,7 +290,7 @@ static int mods_dma_map_default_page(struct mods_client     *client,
 	u64 dev_addr;
 	int err = pci_map_chunk(client, dev, chunk, &dev_addr);
 
-	if (err)
+	if (unlikely(err))
 		return err;
 
 	chunk->dev_addr = dev_addr;
@@ -333,7 +333,7 @@ static int mods_create_default_dma_map(struct mods_client   *client,
 		}
 
 		err = mods_dma_map_default_page(client, chunk, dev);
-		if (err)
+		if (unlikely(err))
 			break;
 	}
 
@@ -361,17 +361,6 @@ static struct MODS_DMA_MAP *find_dma_map(struct MODS_MEM_INFO  *p_mem_info,
 	return NULL;
 }
 
-#if !defined(CONFIG_ARCH_TEGRA) || defined(CONFIG_CPA)
-static int mods_set_mem_type(u64 virt_addr, u64 pages, u8 type)
-{
-	if (type == MODS_ALLOC_UNCACHED)
-		return MODS_SET_MEMORY_UC(virt_addr, pages);
-	else if (type == MODS_ALLOC_WRITECOMBINE)
-		return MODS_SET_MEMORY_WC(virt_addr, pages);
-	return 0;
-}
-#endif
-
 /* In order to map pages as UC or WC to the CPU, we need to change their
  * attributes by calling set_memory_uc()/set_memory_wc(), respectively.
  * On some CPUs this operation is extremely slow.  In order to incur
@@ -387,8 +376,9 @@ static int save_non_wb_chunks(struct mods_client   *client,
 	if (p_mem_info->cache_type == MODS_ALLOC_CACHED)
 		return 0;
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+	err = mutex_lock_interruptible(&client->mtx);
-		return -EINTR;
+	if (unlikely(err))
+		return err;
 
 	/* Steal the chunks from MODS_MEM_INFO and put them on free list. */
 
@@ -403,7 +393,7 @@ static int save_non_wb_chunks(struct mods_client   *client,
 		free_chunk = kzalloc(sizeof(struct MODS_FREE_PHYS_CHUNK),
 				     GFP_KERNEL | __GFP_NORETRY);
 
-		if (!free_chunk) {
+		if (unlikely(!free_chunk)) {
 			err = -ENOMEM;
 			break;
 		}
@@ -449,18 +439,18 @@ static int mods_restore_cache_one_chunk(struct page *p_page, u8 order)
 	u32 i;
 
 	for (i = 0; i < num_pages; i++) {
-		u64 ptr = (u64)(size_t)kmap(p_page + i);
+		void *ptr = kmap(p_page + i);
+		int   err = -ENOMEM;
 
-		if (ptr) {
+		if (likely(ptr))
-			int err = MODS_SET_MEMORY_WB(ptr, 1);
+			err = MODS_SET_MEMORY_WB((unsigned long)ptr, 1);
 
-			if (!final_err)
+		kunmap(ptr);
+
+		if (likely(!final_err))
 			final_err = err;
 	}
 
-		kunmap((void *)(size_t)ptr);
-	}
-
 	return final_err;
 }
 
@@ -471,12 +461,11 @@ static int release_free_chunks(struct mods_client *client)
 	struct list_head *next;
 	int               final_err = 0;
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+	mutex_lock(&client->mtx);
-		return -EINTR;
 
 	head = &client->free_mem_list;
 
-	list_for_each_safe(iter, next, head) {
+	list_for_each_prev_safe(iter, next, head) {
 
 		struct MODS_FREE_PHYS_CHUNK *free_chunk;
 		int                          err;
@@ -489,8 +478,7 @@ static int release_free_chunks(struct mods_client *client)
 
 		err = mods_restore_cache_one_chunk(free_chunk->p_page,
 						   free_chunk->order);
-
+		if (likely(!final_err))
-		if (!final_err)
 			final_err = err;
 
 		__free_pages(free_chunk->p_page, free_chunk->order);
@@ -502,10 +490,14 @@ static int release_free_chunks(struct mods_client *client)
 
 	mutex_unlock(&client->mtx);
 
+	if (unlikely(final_err))
+		cl_error("failed to restore cache attributes\n");
+
 	return final_err;
 }
 
-static int mods_restore_cache(struct MODS_MEM_INFO *p_mem_info)
+static int mods_restore_cache(struct mods_client   *client,
+			      struct MODS_MEM_INFO *p_mem_info)
 {
 	unsigned int i;
 	int          final_err = 0;
@@ -522,10 +514,13 @@ static int mods_restore_cache(struct MODS_MEM_INFO *p_mem_info)
 			continue;
 
 		err = mods_restore_cache_one_chunk(chunk->p_page, chunk->order);
-		if (!final_err)
+		if (likely(!final_err))
 			final_err = err;
 	}
 
+	if (unlikely(final_err))
+		cl_error("failed to restore cache attributes\n");
+
 	return final_err;
 }
 
@@ -534,7 +529,7 @@ static void mods_free_pages(struct mods_client   *client,
 {
 	unsigned int i;
 
-	mods_restore_cache(p_mem_info);
+	mods_restore_cache(client, p_mem_info);
 
 #if defined(CONFIG_ARCH_TEGRA)
 	if (p_mem_info->iommu_mapped)
@@ -599,7 +594,8 @@ static struct page *mods_alloc_pages(struct mods_client   *client,
 	u8           dma32      = p_mem_info->dma32;
 	int          numa_node  = p_mem_info->numa_node;
 
-	if (likely(!mutex_lock_interruptible(&client->mtx))) {
+	if ((cache_type != MODS_MEMORY_CACHED) &&
+	    likely(!mutex_lock_interruptible(&client->mtx))) {
 
 		struct list_head            *iter;
 		struct list_head            *head = &client->free_mem_list;
@@ -647,7 +643,7 @@ static struct page *mods_alloc_pages(struct mods_client   *client,
 
 	*need_cup = 1;
 
-	if (p_page)
+	if (likely(p_page))
 		atomic_add(1u << order, &client->num_pages);
 
 	return p_page;
@@ -656,6 +652,7 @@ static struct page *mods_alloc_pages(struct mods_client   *client,
 static int mods_alloc_contig_sys_pages(struct mods_client   *client,
 				       struct MODS_MEM_INFO *p_mem_info)
 {
+	int          err      = -ENOMEM;
 	u64          phys_addr;
 	u64          dma_addr;
 	u64          end_addr = 0;
@@ -671,10 +668,8 @@ static int mods_alloc_contig_sys_pages(struct mods_client   *client,
 
 	p_page = mods_alloc_pages(client, p_mem_info, order, &is_wb);
 
-	if (!p_page) {
+	if (unlikely(!p_page))
-		LOG_EXT();
+		goto failed;
-		return -ENOMEM;
-	}
 
 	p_mem_info->pages[0].p_page = p_page;
 
@@ -682,20 +677,16 @@ static int mods_alloc_contig_sys_pages(struct mods_client   *client,
 		p_mem_info->pages[0].wc = 1;
 
 	phys_addr = page_to_phys(p_page);
-	if (phys_addr == 0) {
+	if (unlikely(phys_addr == 0)) {
 		cl_error("failed to determine physical address\n");
-		mods_free_pages(client, p_mem_info);
+		goto failed;
-		LOG_EXT();
-		return -ENOMEM;
 	}
 	dma_addr = MODS_PHYS_TO_DMA(phys_addr);
 
-	if (dma_addr >= (1ULL << DMA_BITS)) {
+	if (unlikely(dma_addr >= (1ULL << DMA_BITS))) {
 		cl_error("dma_addr 0x%llx exceeds supported range\n",
 			 dma_addr);
-		mods_free_pages(client, p_mem_info);
+		goto failed;
-		LOG_EXT();
-		return -ENOMEM;
 	}
 
 	p_mem_info->pages[0].dma_addr = dma_addr;
@@ -711,21 +702,16 @@ static int mods_alloc_contig_sys_pages(struct mods_client   *client,
 
 	end_addr = dma_addr +
 		   ((unsigned long)p_mem_info->num_pages << PAGE_SHIFT);
-	if ((p_mem_info->dma32) &&
+	if (unlikely(p_mem_info->dma32 && (end_addr > 0x100000000ULL))) {
-	    (end_addr > 0x100000000ULL)) {
 		cl_error("allocation exceeds 32-bit addressing\n");
-		mods_free_pages(client, p_mem_info);
+		goto failed;
-		LOG_EXT();
-		return -ENOMEM;
 	}
 
-	if (mods_post_alloc(client, p_mem_info->pages, phys_addr, p_mem_info)) {
+	err = mods_post_alloc(client, p_mem_info->pages, phys_addr, p_mem_info);
-		mods_free_pages(client, p_mem_info);
+
+failed:
 	LOG_EXT();
-		return -EINVAL;
+	return err;
-	}
-	LOG_EXT();
-	return 0;
 }
 
 static u32 mods_get_max_order_needed(u32 num_pages)
@@ -740,6 +726,7 @@ static u32 mods_get_max_order_needed(u32 num_pages)
 static int mods_alloc_noncontig_sys_pages(struct mods_client   *client,
 					  struct MODS_MEM_INFO *p_mem_info)
 {
+	int err;
 	u32 pages_left = p_mem_info->num_pages;
 	u32 num_chunks = 0;
 
@@ -748,7 +735,6 @@ static int mods_alloc_noncontig_sys_pages(struct mods_client   *client,
 	memset(p_mem_info->pages, 0,
 	       p_mem_info->num_chunks * sizeof(p_mem_info->pages[0]));
 
-	/* alloc pages */
 	while (pages_left > 0) {
 		u64 phys_addr = 0;
 		u64 dma_addr  = 0;
@@ -756,6 +742,13 @@ static int mods_alloc_noncontig_sys_pages(struct mods_client   *client,
 		int is_wb     = 1;
 		struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[num_chunks];
 
+		/* Fail if memory fragmentation is very high */
+		if (unlikely(num_chunks >= p_mem_info->num_chunks)) {
+			cl_error("detected high memory fragmentation\n");
+			err = -ENOMEM;
+			goto failed;
+		}
+
 		for (;;) {
 			chunk->p_page = mods_alloc_pages(client,
 							 p_mem_info,
@@ -768,8 +761,9 @@ static int mods_alloc_noncontig_sys_pages(struct mods_client   *client,
 			--order;
 		}
 
-		if (!chunk->p_page) {
+		if (unlikely(!chunk->p_page)) {
 			cl_error("out of memory\n");
+			err = -ENOMEM;
 			goto failed;
 		}
 
@@ -780,15 +774,17 @@ static int mods_alloc_noncontig_sys_pages(struct mods_client   *client,
 		chunk->order = order;
 
 		phys_addr = page_to_phys(chunk->p_page);
-		if (phys_addr == 0) {
+		if (unlikely(phys_addr == 0)) {
 			cl_error("phys addr lookup failed\n");
+			err = -ENOMEM;
 			goto failed;
 		}
 		dma_addr = MODS_PHYS_TO_DMA(phys_addr);
 
-		if (dma_addr >= (1ULL << DMA_BITS)) {
+		if (unlikely(dma_addr >= (1ULL << DMA_BITS))) {
 			cl_error("dma_addr 0x%llx exceeds supported range\n",
 				 dma_addr);
+			err = -ENOMEM;
 			goto failed;
 		}
 
@@ -805,23 +801,25 @@ static int mods_alloc_noncontig_sys_pages(struct mods_client   *client,
 
 		++num_chunks;
 
-		if (mods_post_alloc(client, chunk, phys_addr, p_mem_info))
+		err = mods_post_alloc(client, chunk, phys_addr, p_mem_info);
+		if (unlikely(err))
 			goto failed;
 	}
 
-	return 0;
+	err = 0;
 
 failed:
-	mods_free_pages(client, p_mem_info);
+	LOG_EXT();
-	return -ENOMEM;
+	return err;
 }
 
 static int mods_register_alloc(struct mods_client   *client,
 			       struct MODS_MEM_INFO *p_mem_info)
 {
-	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+	int err = mutex_lock_interruptible(&client->mtx);
-		return -EINTR;
 
+	if (unlikely(err))
+		return err;
 	list_add(&p_mem_info->list, &client->mem_alloc_list);
 	mutex_unlock(&client->mtx);
 	return OK;
@@ -853,8 +851,7 @@ static int mods_unregister_and_free(struct mods_client   *client,
 
 	cl_debug(DEBUG_MEM_DETAILED, "free %p\n", p_del_mem);
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx)))
+	mutex_lock(&client->mtx);
-		return -EINTR;
 
 	head = &client->mem_alloc_list;
 
@@ -898,12 +895,12 @@ int mods_unregister_all_alloc(struct mods_client *client)
 
 		p_mem_info = list_entry(iter, struct MODS_MEM_INFO, list);
 		err = mods_unregister_and_free(client, p_mem_info);
-		if (!final_err)
+		if (likely(!final_err))
 			final_err = err;
 	}
 
 	err = release_free_chunks(client);
-	if (!final_err)
+	if (likely(!final_err))
 		final_err = err;
 
 	return final_err;
@@ -1060,7 +1057,10 @@ struct MODS_MEM_INFO *mods_find_alloc(struct mods_client *client, u64 phys_addr)
 	return NULL;
 }
 
-static u32 mods_estimate_num_chunks(u32 num_pages)
+/* Estimate the initial number of chunks supported, assuming medium memory
+ * fragmentation.
+ */
+static u32 estimate_num_chunks(u32 num_pages)
 {
 	u32 num_chunks = 0;
 	u32 bit_scan;
@@ -1079,8 +1079,6 @@ static u32 mods_estimate_num_chunks(u32 num_pages)
 	if (num_chunks > num_pages)
 		num_chunks = num_pages;
 
-	/* Now, if memory is heavily fragmented, we are screwed */
-
 	return num_chunks;
 }
 
@@ -1111,12 +1109,10 @@ static struct MODS_MEM_INFO *optimize_chunks(struct mods_client   *client,
 	}
 
 	if (p_new_mem_info) {
-		atomic_inc(&client->num_allocs);
 		memcpy(p_new_mem_info, p_mem_info, alloc_size);
 		p_new_mem_info->num_chunks = num_chunks;
 		INIT_LIST_HEAD(&p_new_mem_info->dma_map_list);
 		kfree(p_mem_info);
-		atomic_dec(&client->num_allocs);
 		p_mem_info = p_new_mem_info;
 	}
 
@@ -1130,14 +1126,16 @@ static struct MODS_MEM_INFO *optimize_chunks(struct mods_client   *client,
 int esc_mods_alloc_pages_2(struct mods_client        *client,
 			   struct MODS_ALLOC_PAGES_2 *p)
 {
+	int                   err        = -EINVAL;
 	struct MODS_MEM_INFO *p_mem_info = NULL;
 	u32                   num_pages;
 	u32                   alloc_size;
 	u32                   num_chunks;
-	int                   err = OK;
 
 	LOG_ENT();
 
+	p->memory_handle = 0;
+
 	cl_debug(DEBUG_MEM_DETAILED,
 		 "alloc 0x%llx bytes flags=0x%x (%s %s%s%s%s%s) node=%d on dev %04x:%02x:%02x.%x\n",
 		 (unsigned long long)p->num_bytes,
@@ -1155,9 +1153,8 @@ int esc_mods_alloc_pages_2(struct mods_client        *client,
 		 p->pci_device.device,
 		 p->pci_device.function);
 
-	if (!p->num_bytes) {
+	if (unlikely(!p->num_bytes)) {
 		cl_error("zero bytes requested\n");
-		err = -EINVAL;
 		goto failed;
 	}
 
@@ -1165,29 +1162,27 @@ int esc_mods_alloc_pages_2(struct mods_client        *client,
 	if (p->flags & MODS_ALLOC_CONTIGUOUS)
 		num_chunks = 1;
 	else
-		num_chunks = mods_estimate_num_chunks(num_pages);
+		num_chunks = estimate_num_chunks(num_pages);
 	alloc_size = sizeof(*p_mem_info) +
 		     (num_chunks - 1) * sizeof(struct MODS_PHYS_CHUNK);
 
-	if (((u64)num_pages << PAGE_SHIFT) < p->num_bytes) {
+	if (unlikely(((u64)num_pages << PAGE_SHIFT) < p->num_bytes)) {
 		cl_error("invalid allocation size requested: 0x%llx\n",
 			 (unsigned long long)p->num_bytes);
-		err = -EINVAL;
 		goto failed;
 	}
 
-	if ((p->flags & MODS_ALLOC_USE_NUMA) &&
+	if (unlikely((p->flags & MODS_ALLOC_USE_NUMA) &&
 		     (p->numa_node != MODS_ANY_NUMA_NODE) &&
 		      ((unsigned int)p->numa_node >=
-		    (unsigned int)num_possible_nodes())) {
+		       (unsigned int)num_possible_nodes()))) {
 
 		cl_error("invalid NUMA node: %d\n", p->numa_node);
-		err = -EINVAL;
 		goto failed;
 	}
 
 #ifdef CONFIG_PPC64
-	if ((p->flags & MODS_ALLOC_CACHE_MASK) != MODS_ALLOC_CACHED) {
+	if (unlikely((p->flags & MODS_ALLOC_CACHE_MASK) != MODS_ALLOC_CACHED)) {
 		cl_error("unsupported cache attr %u (%s)\n",
 			 p->flags & MODS_ALLOC_CACHE_MASK,
 			 mods_get_prot_str(p->flags & MODS_ALLOC_CACHE_MASK));
@@ -1270,18 +1265,16 @@ int esc_mods_alloc_pages_2(struct mods_client        *client,
 	}
 #endif
 
-	p->memory_handle = 0;
-
 	if (p->flags & MODS_ALLOC_CONTIGUOUS)
 		err = mods_alloc_contig_sys_pages(client, p_mem_info);
 	else {
 		err = mods_alloc_noncontig_sys_pages(client, p_mem_info);
 
-		if (!err)
+		if (likely(!err))
 			p_mem_info = optimize_chunks(client, p_mem_info);
 	}
 
-	if (err) {
+	if (unlikely(err)) {
 		cl_error("failed to alloc 0x%lx %s bytes, %s, node %d%s\n",
 			 (unsigned long)p_mem_info->num_pages << PAGE_SHIFT,
 			 (p->flags & MODS_ALLOC_CONTIGUOUS) ? "contiguous" :
@@ -1292,12 +1285,14 @@ int esc_mods_alloc_pages_2(struct mods_client        *client,
 		goto failed;
 	}
 
+	err = mods_register_alloc(client, p_mem_info);
+	if (unlikely(err))
+		goto failed;
+
 	p->memory_handle = (u64)(size_t)p_mem_info;
 
 	cl_debug(DEBUG_MEM_DETAILED, "alloc %p\n", p_mem_info);
 
-	err = mods_register_alloc(client, p_mem_info);
-
 failed:
 	if (unlikely(err && p_mem_info)) {
 		mods_free_pages(client, p_mem_info);
@@ -1329,7 +1324,7 @@ int esc_mods_device_alloc_pages_2(struct mods_client               *client,
 		flags |= MODS_ALLOC_UNCACHED;
 	else if (p->attrib == MODS_MEMORY_WRITECOMBINE)
 		flags |= MODS_ALLOC_WRITECOMBINE;
-	else if (p->attrib != MODS_MEMORY_CACHED) {
+	else if (unlikely(p->attrib != MODS_MEMORY_CACHED)) {
 		cl_error("invalid cache attrib: %u\n", p->attrib);
 		LOG_EXT();
 		return -ENOMEM;
@@ -1346,7 +1341,7 @@ int esc_mods_device_alloc_pages_2(struct mods_client               *client,
 	dev_alloc_pages.pci_device = p->pci_device;
 
 	err = esc_mods_alloc_pages_2(client, &dev_alloc_pages);
-	if (!err)
+	if (likely(!err))
 		p->memory_handle = dev_alloc_pages.memory_handle;
 
 	LOG_EXT();
@@ -1372,7 +1367,7 @@ int esc_mods_device_alloc_pages(struct mods_client             *client,
 		flags |= MODS_ALLOC_UNCACHED;
 	else if (p->attrib == MODS_MEMORY_WRITECOMBINE)
 		flags |= MODS_ALLOC_WRITECOMBINE;
-	else if (p->attrib != MODS_MEMORY_CACHED) {
+	else if (unlikely(p->attrib != MODS_MEMORY_CACHED)) {
 		cl_error("invalid cache attrib: %u\n", p->attrib);
 		LOG_EXT();
 		return -ENOMEM;
@@ -1392,7 +1387,7 @@ int esc_mods_device_alloc_pages(struct mods_client             *client,
 	dev_alloc_pages.pci_device.function = p->pci_device.function;
 
 	err = esc_mods_alloc_pages_2(client, &dev_alloc_pages);
-	if (!err)
+	if (likely(!err))
 		p->memory_handle = dev_alloc_pages.memory_handle;
 
 	LOG_EXT();
@@ -1417,7 +1412,7 @@ int esc_mods_alloc_pages(struct mods_client *client, struct MODS_ALLOC_PAGES *p)
 		flags |= MODS_ALLOC_UNCACHED;
 	else if (p->attrib == MODS_MEMORY_WRITECOMBINE)
 		flags |= MODS_ALLOC_WRITECOMBINE;
-	else if (p->attrib != MODS_MEMORY_CACHED) {
+	else if (unlikely(p->attrib != MODS_MEMORY_CACHED)) {
 		cl_error("invalid cache attrib: %u\n", p->attrib);
 		LOG_EXT();
 		return -ENOMEM;
@@ -1432,7 +1427,7 @@ int esc_mods_alloc_pages(struct mods_client *client, struct MODS_ALLOC_PAGES *p)
 	dev_alloc_pages.pci_device.function = 0xFFFFU;
 
 	err = esc_mods_alloc_pages_2(client, &dev_alloc_pages);
-	if (!err)
+	if (likely(!err))
 		p->memory_handle = dev_alloc_pages.memory_handle;
 
 	LOG_EXT();
@@ -1472,9 +1467,10 @@ int esc_mods_merge_pages(struct mods_client      *client,
 		return -EINVAL;
 	}
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
 		LOG_EXT();
-		return -EINTR;
+		return err;
 	}
 
 	{
@@ -1620,6 +1616,7 @@ int esc_mods_set_mem_type(struct mods_client      *client,
 {
 	struct MODS_MEM_INFO *p_mem_info;
 	u8                    type = MODS_ALLOC_CACHED;
+	int                   err;
 
 	LOG_ENT();
 
@@ -1641,9 +1638,10 @@ int esc_mods_set_mem_type(struct mods_client      *client,
 		return -EINVAL;
 	}
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
 		LOG_EXT();
-		return -EINTR;
+		return err;
 	}
 
 	p_mem_info = mods_find_alloc(client, p->physical_address);
@@ -1791,12 +1789,14 @@ int esc_mods_virtual_to_phys(struct mods_client              *client,
 	struct MODS_GET_PHYSICAL_ADDRESS get_phys_addr;
 	struct list_head                *head;
 	struct list_head                *iter;
+	int                              err;
 
 	LOG_ENT();
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
 		LOG_EXT();
-		return -EINTR;
+		return err;
 	}
 
 	head = &client->mem_map_list;
@@ -1814,7 +1814,6 @@ int esc_mods_virtual_to_phys(struct mods_client              *client,
 		if (p->virtual_address >= begin && p->virtual_address < end) {
 
 			u64 virt_offs = p->virtual_address - begin;
-			int err;
 
 			/* device memory mapping */
 			if (!p_map_mem->p_mem_info) {
@@ -1844,8 +1843,10 @@ int esc_mods_virtual_to_phys(struct mods_client              *client,
 			mutex_unlock(&client->mtx);
 
 			err = esc_mods_get_phys_addr(client, &get_phys_addr);
-			if (err)
+			if (err) {
+				LOG_EXT();
 				return err;
+			}
 
 			p->physical_address = get_phys_addr.physical_address;
 
@@ -1863,6 +1864,7 @@ int esc_mods_virtual_to_phys(struct mods_client              *client,
 	mutex_unlock(&client->mtx);
 
 	cl_error("invalid virtual address 0x%llx\n", p->virtual_address);
+	LOG_EXT();
 	return -EINVAL;
 }
 
@@ -1874,12 +1876,14 @@ int esc_mods_phys_to_virtual(struct mods_client              *client,
 	struct list_head      *iter;
 	u64                    offset;
 	u64                    map_offset;
+	int                    err;
 
 	LOG_ENT();
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
 		LOG_EXT();
-		return -EINTR;
+		return err;
 	}
 
 	head = &client->mem_map_list;
@@ -1939,8 +1943,11 @@ int esc_mods_phys_to_virtual(struct mods_client              *client,
 			return OK;
 		}
 	}
+
 	mutex_unlock(&client->mtx);
+
 	cl_error("phys addr 0x%llx is not mapped\n", p->physical_address);
+	LOG_EXT();
 	return -EINVAL;
 }
 
@@ -2298,6 +2305,7 @@ static void clear_entry_cache_mappings(struct mods_client    *client,
 			u32 clear_size = PAGE_SIZE - page_offs;
 			u64 remaining  = chunk_offs_end - chunk_offs;
 
+			if (likely(page_va)) {
 				if ((u64)clear_size > remaining)
 					clear_size = (u32)remaining;
 
@@ -2313,6 +2321,8 @@ static void clear_entry_cache_mappings(struct mods_client    *client,
 						       clear_size);
 
 				kunmap((void *)(size_t)page_va);
+			} else
+				cl_error("kmap failed\n");
 
 			chunk_offs += clear_size;
 		}
@@ -2326,18 +2336,23 @@ int esc_mods_flush_cpu_cache_range(struct mods_client                *client,
 {
 	struct list_head *head;
 	struct list_head *iter;
+	int               err;
+
+	LOG_ENT();
 
 	if (irqs_disabled() || in_interrupt() ||
 	    p->virt_addr_start > p->virt_addr_end ||
 	    p->flags == MODS_INVALIDATE_CPU_CACHE) {
 
 		cl_debug(DEBUG_MEM_DETAILED, "cannot clear cache\n");
+		LOG_EXT();
 		return -EINVAL;
 	}
 
-	if (unlikely(mutex_lock_interruptible(&client->mtx))) {
+	err = mutex_lock_interruptible(&client->mtx);
+	if (unlikely(err)) {
 		LOG_EXT();
-		return -EINTR;
+		return err;
 	}
 
 	head = &client->mem_map_list;
@@ -2378,6 +2393,8 @@ int esc_mods_flush_cpu_cache_range(struct mods_client                *client,
 		}
 	}
 	mutex_unlock(&client->mtx);
+
+	LOG_EXT();
 	return OK;
 }
 
@@ -2388,41 +2405,49 @@ static int mods_post_alloc(struct mods_client     *client,
 			   u64                     phys_addr,
 			   struct MODS_MEM_INFO   *p_mem_info)
 {
+	int err = 0;
+
+	if ((p_mem_info->cache_type != MODS_ALLOC_CACHED) && !chunk->wc) {
 		u32 num_pages = 1U << chunk->order;
 		u32 i;
 
-	if (p_mem_info->cache_type != MODS_ALLOC_CACHED && !chunk->wc) {
 		for (i = 0; i < num_pages; i++) {
-			u64 ptr = 0;
+			void *ptr;
-			int err = 0;
 
-			ptr = (u64)(size_t)kmap(chunk->p_page + i);
+			ptr = kmap(chunk->p_page + i);
-			if (!ptr) {
+			if (unlikely(!ptr)) {
 				cl_error("kmap failed\n");
-				return -EINVAL;
+				return -ENOMEM;
 			}
 #if defined(CONFIG_ARCH_TEGRA) && !defined(CONFIG_CPA)
 			clear_contiguous_cache(client,
-					       ptr,
+					       (u64)(size_t)ptr,
 					       phys_addr + (i << PAGE_SHIFT),
 					       PAGE_SIZE);
 #else
-			err = mods_set_mem_type(ptr, 1, p_mem_info->cache_type);
+			if (p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE)
+				err = MODS_SET_MEMORY_WC((unsigned long)ptr, 1);
+			else
+				err = MODS_SET_MEMORY_UC((unsigned long)ptr, 1);
 #endif
-			kunmap((void *)(size_t)ptr);
+			kunmap(ptr);
-			if (err) {
+			if (unlikely(err)) {
 				cl_error("set cache type failed\n");
-				return -EINVAL;
+				return err;
-			}
 			}
 
+			/* Set this flag early, so that when an error occurs,
+			 * mods_free_pages() will restore cache attributes
+			 * for all pages.  It's OK to restore cache attributes
+			 * even for chunks where we haven't change them.
+			 */
 			chunk->wc = 1;
 		}
+	}
 
 #ifdef CONFIG_PCI
 	if (p_mem_info->dev) {
 		struct pci_dev *dev = p_mem_info->dev;
-		int             err;
 
 		/* On systems with SWIOTLB active, disable default DMA mapping
 		 * because we don't support scatter-gather lists.
@@ -2432,13 +2457,11 @@ static int mods_post_alloc(struct mods_client     *client,
 		const struct dma_map_ops *ops = get_dma_ops(&dev->dev);
 
 		if (ops->map_sg == swiotlb_map_sg_attrs)
-			return OK;
+			return 0;
 #endif
 		err = mods_dma_map_default_page(client, chunk, dev);
-		if (err)
-			return err;
 	}
 #endif
 
-	return 0;
+	return err;
 }

drivers/misc/mods/mods_pci.c

@@ -917,3 +917,35 @@ int esc_mods_pci_set_dma_mask(struct mods_client       *client,
 	LOG_EXT();
 	return err;
 }
+
+int esc_mods_pci_reset_function(struct mods_client    *client,
+				struct mods_pci_dev_2 *pcidev)
+{
+	int             err;
+	struct pci_dev *dev;
+
+	LOG_ENT();
+
+	err = mods_find_pci_dev(client, pcidev, &dev);
+	if (unlikely(err)) {
+		if (err == -ENODEV)
+			cl_error("dev %04x:%02x:%02x.%x not found\n",
+				 pcidev->domain,
+				 pcidev->bus,
+				 pcidev->device,
+				 pcidev->function);
+		LOG_EXT();
+		return err;
+	}
+
+	err = pci_reset_function(dev);
+	if (unlikely(err))
+		cl_error("pci_reset_function failed on dev %04x:%02x:%02x.%x\n",
+			 pcidev->domain,
+			 pcidev->bus,
+			 pcidev->device,
+			 pcidev->function);
+	pci_dev_put(dev);
+	LOG_EXT();
+	return err;
+}

include/uapi/misc/mods.h

@@ -25,7 +25,7 @@
 
 /* Driver version */
 #define MODS_DRIVER_VERSION_MAJOR 3
-#define MODS_DRIVER_VERSION_MINOR 97
+#define MODS_DRIVER_VERSION_MINOR 99
 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \
 			     ((MODS_DRIVER_VERSION_MINOR / 10) << 4) | \
 			     (MODS_DRIVER_VERSION_MINOR % 10))
@@ -1024,11 +1024,17 @@ union ACPI_ARGUMENT {
 		__u32 length; /* Number of bytes */
 		__u32 offset; /* Offset in in_buffer or out_buffer */
 	} buffer;
+
+	struct {
+		__u32 type;
+		__u64 handle;
+	} method;
 };
 
 /* Argument type (for the type field above) */
 #define ACPI_MODS_TYPE_INTEGER   1
 #define ACPI_MODS_TYPE_BUFFER    2
+#define ACPI_MODS_TYPE_METHOD    3
 
 #define ACPI_MAX_BUFFER_LENGTH   4096
 #define ACPI_MAX_METHOD_LENGTH   12
@@ -1353,12 +1359,13 @@ struct MODS_GET_NVLINK_LINE_RATE {
 };
 
 #define MODS_MAX_SYSFS_PATH_BUF_SIZE 512
-#define MODS_MAX_SYSFS_PATH_LEN (512 - 6)
 #define MODS_MAX_SYSFS_FILE_SIZE 4096
 
 /* Used by MODS_ESC_WRITE_SYSFS_NODE ioctl.
  *
  * Writes specified contents to the given sysfs node.
+ *
+ * 'path' parameter is relative to /sys/.
  */
 struct MODS_SYSFS_NODE {
 	/* IN */
@@ -1367,6 +1374,18 @@ struct MODS_SYSFS_NODE {
 	__u32 size; /* Size of the contents buffer, in bytes */
 };
 
+/* Used by MODS_ESC_SYSCTL_WRITE_INT ioctl.
+ *
+ * Writes specified integer value into a node under /proc/sys/.
+ *
+ * 'path' parameter is relative to /proc/sys/.
+ */
+struct MODS_SYSCTL_INT {
+	/* IN */
+	char  path[MODS_MAX_SYSFS_PATH_BUF_SIZE];
+	__s64 value;
+};
+
 #define MAX_CLOCK_HANDLE_NAME 64
 
 /* Used by MODS_ESC_GET_CLOCK_HANDLE ioctl.
@@ -1923,5 +1942,7 @@ struct MODS_IOMMU_DMA_MAP_MEMORY {
 					       MODS_IOMMU_DMA_MAP_MEMORY)
 #define MODS_ESC_RESET_ASSERT MODSIO(W, 131, MODS_RESET_HANDLE)
 #define MODS_ESC_GET_RESET_HANDLE MODSIO(WR, 132, MODS_GET_RESET_HANDLE)
+#define MODS_ESC_SYSCTL_WRITE_INT MODSIO(W, 133, MODS_SYSCTL_INT)
+#define MODS_ESC_PCI_RESET_FUNCTION MODSIO(W, 134, mods_pci_dev_2)
 
 #endif /* _UAPI_MODS_H_  */