From 003f79282f6f7ff01968a47308bee02c154fd48a Mon Sep 17 00:00:00 2001 From: Chris Dragan Date: Thu, 18 Jul 2019 05:59:43 -0700 Subject: [PATCH] misc: mods: update MODS kernel driver to 3.93 Change-Id: I6bc81407edbd448867c0b6d979d2eb3920c0e95c Signed-off-by: Chris Dragan Reviewed-on: https://git-master.nvidia.com/r/2156158 Reviewed-by: Automatic_Commit_Validation_User GVS: Gerrit_Virtual_Submit Reviewed-by: Sachin Nikam Reviewed-by: mobile promotions Tested-by: mobile promotions --- drivers/misc/mods/mods.h | 60 ++- drivers/misc/mods/mods_acpi.c | 62 +-- drivers/misc/mods/mods_adsp.c | 11 +- drivers/misc/mods/mods_clock.c | 35 +- drivers/misc/mods/mods_debugfs.c | 11 +- drivers/misc/mods/mods_dma.c | 24 +- drivers/misc/mods/mods_dmabuf.c | 2 +- drivers/misc/mods/mods_internal.h | 325 ++++++------ drivers/misc/mods/mods_irq.c | 288 +++++----- drivers/misc/mods/mods_krnl.c | 360 +++++++------ drivers/misc/mods/mods_mem.c | 812 +++++++++++++++++++++-------- drivers/misc/mods/mods_netdevice.c | 5 +- drivers/misc/mods/mods_pci.c | 144 +++-- drivers/misc/mods/mods_tegradc.c | 4 +- drivers/misc/mods/mods_tegraprod.c | 14 +- 15 files changed, 1303 insertions(+), 854 deletions(-) diff --git a/drivers/misc/mods/mods.h b/drivers/misc/mods/mods.h index b990bacb..873fe682 100644 --- a/drivers/misc/mods/mods.h +++ b/drivers/misc/mods/mods.h @@ -25,7 +25,7 @@ /* Driver version */ #define MODS_DRIVER_VERSION_MAJOR 3 -#define MODS_DRIVER_VERSION_MINOR 91 +#define MODS_DRIVER_VERSION_MINOR 93 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \ ((MODS_DRIVER_VERSION_MINOR/10) << 4) | \ (MODS_DRIVER_VERSION_MINOR%10)) @@ -49,6 +49,40 @@ struct mods_pci_dev { __u8 function; }; +/* MODS_ESC_ALLOC_PAGES_2 */ +struct MODS_ALLOC_PAGES_2 { + /* IN */ + __u64 num_bytes; + __u32 flags; /* MODS_ALLOC_* */ + __s32 numa_node; + struct mods_pci_dev_2 pci_device; + + /* OUT */ + __u64 memory_handle; +}; + +/* numa_node */ +#define MODS_ANY_NUMA_NODE (-1) + +/* flags */ +#define MODS_ALLOC_CACHED 0 /* Default WB cache attr */ +#define MODS_ALLOC_UNCACHED 1 /* UC cache attr */ +#define MODS_ALLOC_WRITECOMBINE 2 /* WC cache attr */ +#define MODS_ALLOC_CACHE_MASK 7U /* The first three bits are cache attr */ + +#define MODS_ALLOC_DMA32 8 /* Force 32-bit PA, else any PA */ +#define MODS_ALLOC_CONTIGUOUS 16 /* Force contiguous, else any PA */ +#define MODS_ALLOC_USE_NUMA 32 /* Use numa_node instead of PCI dev for + * NUMA node hint + */ +#define MODS_ALLOC_FORCE_NUMA 64 /* Force memory to be from a given NUMA + * node (same as PCI dev or numa_node). + * Otherwise use PCI dev/numa_node as + * a hint and if that is not possible, + * allocate from any NUMA node + */ +#define MODS_ALLOC_MAP_DEV 128 /* DMA map to PCI device */ + /* MODS_ESC_ALLOC_PAGES */ struct MODS_ALLOC_PAGES { /* IN */ @@ -317,6 +351,12 @@ struct MODS_PCI_HOT_RESET { struct mods_pci_dev_2 pci_device; }; +/* MODS_ESC_PCI_BUS_REMOVE_DEV */ +struct MODS_PCI_BUS_REMOVE_DEV { + /* IN */ + struct mods_pci_dev_2 pci_device; +}; + /* MODS_ESC_SET_PPC_TCE_BYPASS */ #define MODS_PPC_TCE_BYPASS_DEFAULT 0 #define MODS_PPC_TCE_BYPASS_ON 1 @@ -1128,22 +1168,29 @@ struct MODS_MSR { _IOWR(MODS_IOC_MAGIC, 9, struct MODS_PIO_READ) #define MODS_ESC_PIO_WRITE \ _IOWR(MODS_IOC_MAGIC, 10, struct MODS_PIO_WRITE) +/* Deprecated */ #define MODS_ESC_IRQ_REGISTER \ _IOWR(MODS_IOC_MAGIC, 11, struct MODS_IRQ) +/* Deprecated */ #define MODS_ESC_IRQ_FREE \ _IOWR(MODS_IOC_MAGIC, 12, struct MODS_IRQ) +/* Deprecated */ #define MODS_ESC_IRQ_INQUIRY \ _IOWR(MODS_IOC_MAGIC, 13, struct MODS_IRQ) #define MODS_ESC_EVAL_ACPI_METHOD \ _IOWR_BAD(MODS_IOC_MAGIC, 16, struct MODS_EVAL_ACPI_METHOD) #define MODS_ESC_GET_API_VERSION \ _IOWR(MODS_IOC_MAGIC, 17, struct MODS_GET_VERSION) +/* Deprecated */ #define MODS_ESC_GET_KERNEL_VERSION \ _IOWR(MODS_IOC_MAGIC, 18, struct MODS_GET_VERSION) +/* Deprecated */ #define MODS_ESC_SET_DRIVER_PARA \ _IOWR(MODS_IOC_MAGIC, 19, struct MODS_SET_PARA) +/* Deprecated */ #define MODS_ESC_MSI_REGISTER \ _IOWR(MODS_IOC_MAGIC, 20, struct MODS_IRQ) +/* Deprecated */ #define MODS_ESC_REARM_MSI \ _IOWR(MODS_IOC_MAGIC, 21, struct MODS_IRQ) #define MODS_ESC_SET_MEMORY_TYPE \ @@ -1180,6 +1227,7 @@ struct MODS_MSR { _IOW(MODS_IOC_MAGIC, 37, struct MODS_CLOCK_HANDLE) #define MODS_ESC_CLOCK_RESET_DEASSERT \ _IOW(MODS_IOC_MAGIC, 38, struct MODS_CLOCK_HANDLE) +/* Deprecated */ #define MODS_ESC_SET_IRQ_MASK \ _IOW(MODS_IOC_MAGIC, 39, struct MODS_SET_IRQ_MASK) #define MODS_ESC_MEMORY_BARRIER \ @@ -1216,6 +1264,7 @@ struct MODS_MSR { _IOWR(MODS_IOC_MAGIC, 54, struct MODS_PCI_GET_BAR_INFO) #define MODS_ESC_PCI_GET_IRQ \ _IOWR(MODS_IOC_MAGIC, 55, struct MODS_PCI_GET_IRQ) +/* Deprecated */ #define MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS \ _IOWR(MODS_IOC_MAGIC, 56, \ struct MODS_GET_PHYSICAL_ADDRESS) @@ -1242,6 +1291,7 @@ struct MODS_MSR { _IOW(MODS_IOC_MAGIC, 66, struct MODS_REGISTER_IRQ_2) #define MODS_ESC_QUERY_IRQ_2 \ _IOR(MODS_IOC_MAGIC, 67, struct MODS_QUERY_IRQ_2) +/* Deprecated */ #define MODS_ESC_SET_IRQ_MASK_2 \ _IOW(MODS_IOC_MAGIC, 68, struct MODS_SET_IRQ_MASK_2) #define MODS_ESC_EVAL_DEV_ACPI_METHOD_2 \ @@ -1261,11 +1311,14 @@ struct MODS_MSR { _IOW(MODS_IOC_MAGIC, 75, struct MODS_DMA_MAP_MEMORY) #define MODS_ESC_DMA_UNMAP_MEMORY \ _IOW(MODS_IOC_MAGIC, 76, struct MODS_DMA_MAP_MEMORY) +/* Deprecated */ #define MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_2 \ _IOWR(MODS_IOC_MAGIC, 77, \ struct MODS_GET_PHYSICAL_ADDRESS_2) +/* Deprecated */ #define MODS_ESC_PCI_MAP_RESOURCE \ _IOWR(MODS_IOC_MAGIC, 78, struct MODS_PCI_MAP_RESOURCE) +/* Deprecated */ #define MODS_ESC_PCI_UNMAP_RESOURCE \ _IOW(MODS_IOC_MAGIC, 79, struct MODS_PCI_UNMAP_RESOURCE) #define MODS_ESC_DMA_REQUEST_HANDLE \ @@ -1280,6 +1333,7 @@ struct MODS_MSR { _IOWR(MODS_IOC_MAGIC, 84, struct MODS_DMA_WAIT_DESC) #define MODS_ESC_DMA_ISSUE_PENDING \ _IOW(MODS_IOC_MAGIC, 85, struct MODS_DMA_HANDLE) +/* Deprecated */ #define MODS_ESC_SET_IRQ_MULTIMASK \ _IOW(MODS_IOC_MAGIC, 86, struct MODS_SET_IRQ_MULTIMASK) #define MODS_ESC_NET_FORCE_LINK \ @@ -1370,5 +1424,9 @@ struct MODS_MSR { #define MODS_ESC_EVAL_DEV_ACPI_METHOD_3 \ _IOWR_BAD(MODS_IOC_MAGIC, 122,\ struct MODS_EVAL_DEV_ACPI_METHOD_3) +#define MODS_ESC_PCI_BUS_REMOVE_DEV\ + _IOW(MODS_IOC_MAGIC, 123, struct MODS_PCI_BUS_REMOVE_DEV) +#define MODS_ESC_ALLOC_PAGES_2 \ + _IOWR(MODS_IOC_MAGIC, 124, struct MODS_ALLOC_PAGES_2) #endif /* _MODS_H_ */ diff --git a/drivers/misc/mods/mods_acpi.c b/drivers/misc/mods/mods_acpi.c index 3c11a4bc..5ae360e4 100644 --- a/drivers/misc/mods/mods_acpi.c +++ b/drivers/misc/mods/mods_acpi.c @@ -155,10 +155,10 @@ static int mods_extract_acpi_object( return err; } -static int mods_eval_acpi_method(struct file *pfile, +static int mods_eval_acpi_method(struct mods_client *client, struct MODS_EVAL_ACPI_METHOD *p, - struct mods_pci_dev_2 *pdevice, - u32 acpi_id) + struct mods_pci_dev_2 *pdevice, + u32 acpi_id) { int err = OK; int i; @@ -190,7 +190,7 @@ static int mods_eval_acpi_method(struct file *pfile, pdevice->device, pdevice->function); - err = mods_find_pci_dev(pfile, pdevice, &dev); + err = mods_find_pci_dev(client, pdevice, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("ACPI: PCI device %04x:%02x:%02x.%x not found\n", @@ -219,12 +219,7 @@ static int mods_eval_acpi_method(struct file *pfile, acpi_method_handler = NULL; list_for_each_safe(node, next, &acpi_dev->children) { -#ifdef MODS_ACPI_DEVID_64 - unsigned long long -#else - unsigned long -#endif - device_id = 0; + unsigned long long device_id = 0; struct acpi_device *acpi_dev = list_entry(node, struct acpi_device, node); @@ -240,11 +235,7 @@ static int mods_eval_acpi_method(struct file *pfile, /* Couldn't query device_id for this device */ continue; -#ifdef MODS_ACPI_DEVID_64 if (device_id == acpi_id) { -#else - if ((device_id & 0xffff) == (acpi_id & 0xffff)) { -#endif acpi_method_handler = acpi_dev->handle; mods_debug_printk(DEBUG_ACPI, "ACPI: Found %s (id = 0x%x) on device %04x:%02x:%02x.%x\n", @@ -328,9 +319,9 @@ static int mods_eval_acpi_method(struct file *pfile, return err; } -static int mods_acpi_get_ddc(struct file *pfile, +static int mods_acpi_get_ddc(struct mods_client *client, struct MODS_ACPI_GET_DDC_2 *p, - struct mods_pci_dev_2 *pci_device) + struct mods_pci_dev_2 *pci_device) { int err; acpi_status status; @@ -355,7 +346,7 @@ static int mods_acpi_get_ddc(struct file *pfile, pci_device->device, pci_device->function); - err = mods_find_pci_dev(pfile, pci_device, &dev); + err = mods_find_pci_dev(client, pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("ACPI: PCI device %04x:%02x:%02x.%x not found\n", @@ -385,12 +376,7 @@ static int mods_acpi_get_ddc(struct file *pfile, } list_for_each_safe(node, next, &device->children) { -#ifdef MODS_ACPI_DEVID_64 - unsigned long long -#else - unsigned long -#endif - device_id = 0; + unsigned long long device_id = 0; struct acpi_device *dev = list_entry(node, struct acpi_device, node); @@ -489,26 +475,31 @@ static int mods_acpi_get_ddc(struct file *pfile, * ESCAPE CALL FUNCTIONS * *************************/ -int esc_mods_eval_acpi_method(struct file *pfile, +int esc_mods_eval_acpi_method(struct mods_client *client, struct MODS_EVAL_ACPI_METHOD *p) { - return mods_eval_acpi_method(pfile, p, 0, ACPI_MODS_IGNORE_ACPI_ID); + return mods_eval_acpi_method(client, p, 0, ACPI_MODS_IGNORE_ACPI_ID); } -int esc_mods_eval_dev_acpi_method_3(struct file *pfile, +int esc_mods_eval_dev_acpi_method_3(struct mods_client *client, struct MODS_EVAL_DEV_ACPI_METHOD_3 *p) { - return mods_eval_acpi_method(pfile, &p->method, &p->device, p->acpi_id); + return mods_eval_acpi_method(client, + &p->method, + &p->device, + p->acpi_id); } -int esc_mods_eval_dev_acpi_method_2(struct file *pfile, +int esc_mods_eval_dev_acpi_method_2(struct mods_client *client, struct MODS_EVAL_DEV_ACPI_METHOD_2 *p) { - return mods_eval_acpi_method(pfile, &p->method, &p->device, + return mods_eval_acpi_method(client, + &p->method, + &p->device, ACPI_MODS_IGNORE_ACPI_ID); } -int esc_mods_eval_dev_acpi_method(struct file *pfile, +int esc_mods_eval_dev_acpi_method(struct mods_client *client, struct MODS_EVAL_DEV_ACPI_METHOD *p) { struct mods_pci_dev_2 device = {0}; @@ -517,17 +508,18 @@ int esc_mods_eval_dev_acpi_method(struct file *pfile, device.bus = p->device.bus; device.device = p->device.device; device.function = p->device.function; - return mods_eval_acpi_method(pfile, &p->method, &device, + return mods_eval_acpi_method(client, &p->method, &device, ACPI_MODS_IGNORE_ACPI_ID); } -int esc_mods_acpi_get_ddc_2(struct file *pfile, +int esc_mods_acpi_get_ddc_2(struct mods_client *client, struct MODS_ACPI_GET_DDC_2 *p) { - return mods_acpi_get_ddc(pfile, p, &p->device); + return mods_acpi_get_ddc(client, p, &p->device); } -int esc_mods_acpi_get_ddc(struct file *pfile, struct MODS_ACPI_GET_DDC *p) +int esc_mods_acpi_get_ddc(struct mods_client *client, + struct MODS_ACPI_GET_DDC *p) { struct MODS_ACPI_GET_DDC_2 *pp = (struct MODS_ACPI_GET_DDC_2 *) p; struct mods_pci_dev_2 device = {0}; @@ -537,5 +529,5 @@ int esc_mods_acpi_get_ddc(struct file *pfile, struct MODS_ACPI_GET_DDC *p) device.device = p->device.device; device.function = p->device.function; - return mods_acpi_get_ddc(pfile, pp, &device); + return mods_acpi_get_ddc(client, pp, &device); } diff --git a/drivers/misc/mods/mods_adsp.c b/drivers/misc/mods/mods_adsp.c index 0ed04e8c..29fb9409 100644 --- a/drivers/misc/mods/mods_adsp.c +++ b/drivers/misc/mods/mods_adsp.c @@ -1,7 +1,7 @@ /* * mods_adsp.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -21,22 +21,23 @@ #include "mods_internal.h" #include -int esc_mods_adsp_load(struct file *pfile) +int esc_mods_adsp_load(struct mods_client *client) { return nvadsp_os_load(); } -int esc_mods_adsp_start(struct file *pfile) +int esc_mods_adsp_start(struct mods_client *client) { return nvadsp_os_start(); } -int esc_mods_adsp_stop(struct file *pfile) +int esc_mods_adsp_stop(struct mods_client *client) { return nvadsp_os_suspend(); } -int esc_mods_adsp_run_app(struct file *pfile, struct MODS_ADSP_RUN_APP_INFO *p) +int esc_mods_adsp_run_app(struct mods_client *client, + struct MODS_ADSP_RUN_APP_INFO *p) { int rc = -1; int max_retry = 3; diff --git a/drivers/misc/mods/mods_clock.c b/drivers/misc/mods/mods_clock.c index f64aaf29..e2ae85cd 100644 --- a/drivers/misc/mods/mods_clock.c +++ b/drivers/misc/mods/mods_clock.c @@ -1,7 +1,7 @@ /* * mods_clock.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2011-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -163,7 +163,7 @@ static struct clk *mods_get_clock(u32 handle) return pclk; } -int esc_mods_get_clock_handle(struct file *pfile, +int esc_mods_get_clock_handle(struct mods_client *client, struct MODS_GET_CLOCK_HANDLE *p) { struct clk *pclk = 0; @@ -201,7 +201,8 @@ err: return ret; } -int esc_mods_set_clock_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) +int esc_mods_set_clock_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -230,7 +231,8 @@ int esc_mods_set_clock_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) return ret; } -int esc_mods_get_clock_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) +int esc_mods_get_clock_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -253,7 +255,8 @@ int esc_mods_get_clock_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) return ret; } -int esc_mods_get_clock_max_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) +int esc_mods_get_clock_max_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -279,7 +282,8 @@ int esc_mods_get_clock_max_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) return ret; } -int esc_mods_set_clock_max_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) +int esc_mods_set_clock_max_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -315,7 +319,8 @@ int esc_mods_set_clock_max_rate(struct file *pfile, struct MODS_CLOCK_RATE *p) return ret; } -int esc_mods_set_clock_parent(struct file *pfile, struct MODS_CLOCK_PARENT *p) +int esc_mods_set_clock_parent(struct mods_client *client, + struct MODS_CLOCK_PARENT *p) { struct clk *pclk = 0; struct clk *pparent = 0; @@ -349,7 +354,8 @@ int esc_mods_set_clock_parent(struct file *pfile, struct MODS_CLOCK_PARENT *p) return ret; } -int esc_mods_get_clock_parent(struct file *pfile, struct MODS_CLOCK_PARENT *p) +int esc_mods_get_clock_parent(struct mods_client *client, + struct MODS_CLOCK_PARENT *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -375,7 +381,8 @@ int esc_mods_get_clock_parent(struct file *pfile, struct MODS_CLOCK_PARENT *p) return ret; } -int esc_mods_enable_clock(struct file *pfile, struct MODS_CLOCK_HANDLE *p) +int esc_mods_enable_clock(struct mods_client *client, + struct MODS_CLOCK_HANDLE *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -408,7 +415,8 @@ int esc_mods_enable_clock(struct file *pfile, struct MODS_CLOCK_HANDLE *p) return ret; } -int esc_mods_disable_clock(struct file *pfile, struct MODS_CLOCK_HANDLE *p) +int esc_mods_disable_clock(struct mods_client *client, + struct MODS_CLOCK_HANDLE *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -432,7 +440,8 @@ int esc_mods_disable_clock(struct file *pfile, struct MODS_CLOCK_HANDLE *p) return ret; } -int esc_mods_is_clock_enabled(struct file *pfile, struct MODS_CLOCK_ENABLED *p) +int esc_mods_is_clock_enabled(struct mods_client *client, + struct MODS_CLOCK_ENABLED *p) { struct clk *pclk = 0; int ret = -EINVAL; @@ -455,7 +464,7 @@ int esc_mods_is_clock_enabled(struct file *pfile, struct MODS_CLOCK_ENABLED *p) return ret; } -int esc_mods_clock_reset_assert(struct file *pfile, +int esc_mods_clock_reset_assert(struct mods_client *client, struct MODS_CLOCK_HANDLE *p) { struct clk *pclk = 0; @@ -511,7 +520,7 @@ err: return ret; } -int esc_mods_clock_reset_deassert(struct file *pfile, +int esc_mods_clock_reset_deassert(struct mods_client *client, struct MODS_CLOCK_HANDLE *p) { struct clk *pclk = 0; diff --git a/drivers/misc/mods/mods_debugfs.c b/drivers/misc/mods/mods_debugfs.c index 6b04bb18..3f85765c 100644 --- a/drivers/misc/mods/mods_debugfs.c +++ b/drivers/misc/mods/mods_debugfs.c @@ -2,7 +2,7 @@ /* * mods_debugfs.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -20,8 +20,6 @@ #include "mods_internal.h" -#ifdef MODS_HAS_DEBUGFS - #include #include #include @@ -477,19 +475,15 @@ static int mods_mi_set(void *data, u64 val) return 0; } DEFINE_SIMPLE_ATTRIBUTE(mods_mi_fops, mods_mi_get, mods_mi_set, "%llu\n"); -#endif /* MODS_HAS_DEBUGFS */ void mods_remove_debugfs(void) { -#ifdef MODS_HAS_DEBUGFS debugfs_remove_recursive(mods_debugfs_dir); mods_debugfs_dir = NULL; -#endif } int mods_create_debugfs(struct miscdevice *modsdev) { -#ifdef MODS_HAS_DEBUGFS struct dentry *retval; int err = 0; #ifdef CONFIG_TEGRA_DC @@ -699,8 +693,5 @@ remove_out: dev_err(modsdev->this_device, "could not create debugfs\n"); mods_remove_debugfs(); return err; -#else - return 0; -#endif } diff --git a/drivers/misc/mods/mods_dma.c b/drivers/misc/mods/mods_dma.c index 2fd58146..15fac63b 100644 --- a/drivers/misc/mods/mods_dma.c +++ b/drivers/misc/mods/mods_dma.c @@ -1,7 +1,7 @@ /* * mods_dma.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -198,7 +198,7 @@ int mods_dma_async_is_tx_complete(struct MODS_DMA_HANDLE *p_handle, return ret; } -int esc_mods_dma_request_channel(struct file *pfile, +int esc_mods_dma_request_channel(struct mods_client *client, struct MODS_DMA_HANDLE *p_handle) { struct dma_chan *chan; @@ -251,14 +251,14 @@ int esc_mods_dma_request_channel(struct file *pfile, return 0; } -int esc_mods_dma_release_channel(struct file *pfile, +int esc_mods_dma_release_channel(struct mods_client *client, struct MODS_DMA_HANDLE *p_handle) { mods_release_channel(p_handle->dma_id); return OK; } -int esc_mods_dma_set_config(struct file *pfile, +int esc_mods_dma_set_config(struct mods_client *client, struct MODS_DMA_CHANNEL_CONFIG *p_config) { @@ -302,7 +302,7 @@ int esc_mods_dma_set_config(struct file *pfile, } -int esc_mods_dma_submit_request(struct file *pfile, +int esc_mods_dma_submit_request(struct mods_client *client, struct MODS_DMA_TX_DESC *p_mods_desc) { int ret = OK; @@ -373,7 +373,7 @@ failed: return ret; } -int esc_mods_dma_async_issue_pending(struct file *pfile, +int esc_mods_dma_async_issue_pending(struct mods_client *client, struct MODS_DMA_HANDLE *p_handle) { int ret = OK; @@ -394,7 +394,7 @@ int esc_mods_dma_async_issue_pending(struct file *pfile, return ret; } -int esc_mods_dma_wait(struct file *pfile, +int esc_mods_dma_wait(struct mods_client *client, struct MODS_DMA_WAIT_DESC *p_wait_desc) { int ret; @@ -414,7 +414,7 @@ int esc_mods_dma_wait(struct file *pfile, return ret; } -int esc_mods_dma_alloc_coherent(struct file *fp, +int esc_mods_dma_alloc_coherent(struct mods_client *client, struct MODS_DMA_COHERENT_MEM_HANDLE *p) { dma_addr_t p_phys_addr; @@ -454,8 +454,8 @@ int esc_mods_dma_alloc_coherent(struct file *fp, return 0; } -int esc_mods_dma_free_coherent(struct file *fp, - struct MODS_DMA_COHERENT_MEM_HANDLE *p) +int esc_mods_dma_free_coherent(struct mods_client *client, + struct MODS_DMA_COHERENT_MEM_HANDLE *p) { LOG_ENT(); @@ -478,8 +478,8 @@ int esc_mods_dma_free_coherent(struct file *fp, return 0; } -int esc_mods_dma_copy_to_user(struct file *fp, - struct MODS_DMA_COPY_TO_USER *p) +int esc_mods_dma_copy_to_user(struct mods_client *client, + struct MODS_DMA_COPY_TO_USER *p) { int retval; diff --git a/drivers/misc/mods/mods_dmabuf.c b/drivers/misc/mods/mods_dmabuf.c index 9208e3e6..1e48ae8b 100644 --- a/drivers/misc/mods/mods_dmabuf.c +++ b/drivers/misc/mods/mods_dmabuf.c @@ -37,7 +37,7 @@ static struct device *dummy_device; static bool dummy_device_registered; -int esc_mods_dmabuf_get_phys_addr(struct file *filp, +int esc_mods_dmabuf_get_phys_addr(struct mods_client *client, struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *op) { int err = 0; diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h index 9869faf8..ab5eb4c8 100644 --- a/drivers/misc/mods/mods_internal.h +++ b/drivers/misc/mods/mods_internal.h @@ -65,7 +65,7 @@ struct en_dev_entry { struct mem_type { u64 dma_addr; u64 size; - u32 type; + u8 type; }; struct irq_q_data { @@ -88,6 +88,7 @@ struct mods_client { struct list_head irq_list; struct list_head mem_alloc_list; struct list_head mem_map_list; + struct list_head free_mem_list; /* list of unused UC/WC chunks */ #if defined(CONFIG_PPC64) struct list_head ppc_tce_bypass_list; struct list_head nvlink_sysmem_trained_list; @@ -103,13 +104,15 @@ struct mods_client { struct mutex mtx; int mods_fb_suspended[FB_MAX]; u32 access_token; + atomic_t num_allocs; + atomic_t num_pages; u8 client_id; }; /* VM private data */ struct mods_vm_private_data { - struct file *fp; - atomic_t usage_count; + struct mods_client *client; + atomic_t usage_count; }; /* PCI Resource mapping private data*/ @@ -120,14 +123,26 @@ struct MODS_PCI_RES_MAP_INFO { struct list_head list; }; +#define DMA_BITS 57 + struct MODS_PHYS_CHUNK { - u64 dma_addr:58; /* phys addr (or machine addr on XEN) */ - u32 order:5; /* 1<vm_pgoff) #define MODS_VMA_SIZE(vma) ((vma)->vm_end - (vma)->vm_start) #define MODS_VMA_OFFSET(vma) (((u64)(vma)->vm_pgoff) << PAGE_SHIFT) -#define MODS_VMA_PRIVATE(vma) ((vma)->vm_private_data) #define MODS_VMA_FILE(vma) ((vma)->vm_file) /* Xen adds a translation layer between the physical address @@ -355,22 +351,6 @@ struct mods_priv { #define MODS_ACPI_HANDLE(dev) DEVICE_ACPI_HANDLE(dev) #endif -#ifdef MODS_HAS_OLD_PCI_DMA_MAPPING_ERROR -#define MODS_PCI_DMA_MAPPING_ERROR(dev, addr) pci_dma_mapping_error(addr) -#else -#define MODS_PCI_DMA_MAPPING_ERROR(dev, addr) pci_dma_mapping_error(dev, addr) -#endif - -static inline u8 get_client_id(struct file *fp) -{ - return ((struct mods_client *)(fp->private_data))->client_id; -} - -static inline int is_client_id_valid(u8 client_id) -{ - return client_id > 0 && client_id <= MODS_MAX_CLIENTS; -} - /* ************************************************************************* */ /* ** MODULE WIDE FUNCTIONS */ /* ************************************************************************* */ @@ -385,28 +365,30 @@ void mods_free_client(u8 client_id); int mods_irq_event_check(u8 client_id); /* mem */ -const char *mods_get_prot_str(u32 mem_type); -int mods_unregister_all_alloc(struct file *fp); -struct MODS_MEM_INFO *mods_find_alloc(struct file *fp, u64 phys_addr); +const char *mods_get_prot_str(u8 mem_type); +int mods_unregister_all_alloc(struct mods_client *client); +struct MODS_MEM_INFO *mods_find_alloc(struct mods_client *client, + u64 phys_addr); #if defined(CONFIG_PPC64) /* ppc64 */ -int mods_unregister_all_ppc_tce_bypass(struct file *fp); +int mods_unregister_all_ppc_tce_bypass(struct mods_client *client); -int mods_unregister_all_nvlink_sysmem_trained(struct file *fp); +int mods_unregister_all_nvlink_sysmem_trained(struct mods_client *client); #endif /* pci */ #ifdef CONFIG_PCI -struct en_dev_entry *mods_enable_device(struct mods_client *client, - struct pci_dev *dev); +int mods_enable_device(struct mods_client *client, + struct pci_dev *dev, + struct en_dev_entry **dev_entry); void mods_disable_device(struct pci_dev *pdev); #endif #ifdef CONFIG_PCI int mods_is_pci_dev(struct pci_dev *dev, struct mods_pci_dev_2 *pcidev); -int mods_find_pci_dev(struct file *fp, +int mods_find_pci_dev(struct mods_client *client, struct mods_pci_dev_2 *pcidev, struct pci_dev **retdev); #else @@ -423,193 +405,224 @@ void mods_shutdown_clock_api(void); /* ioctl hanndlers */ /* mem */ -int esc_mods_alloc_pages(struct file *fp, struct MODS_ALLOC_PAGES *p); -int esc_mods_device_alloc_pages(struct file *fp, +int esc_mods_alloc_pages(struct mods_client *client, + struct MODS_ALLOC_PAGES *p); +int esc_mods_device_alloc_pages(struct mods_client *client, struct MODS_DEVICE_ALLOC_PAGES *p); -int esc_mods_device_alloc_pages_2(struct file *fp, +int esc_mods_device_alloc_pages_2(struct mods_client *client, struct MODS_DEVICE_ALLOC_PAGES_2 *p); -int esc_mods_free_pages(struct file *fp, struct MODS_FREE_PAGES *p); -int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p); -int esc_mods_get_phys_addr(struct file *fp, +int esc_mods_alloc_pages_2(struct mods_client *client, + struct MODS_ALLOC_PAGES_2 *p); +int esc_mods_free_pages(struct mods_client *client, + struct MODS_FREE_PAGES *p); +int esc_mods_set_mem_type(struct mods_client *client, + struct MODS_MEMORY_TYPE *p); +int esc_mods_get_phys_addr(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS *p); -int esc_mods_get_phys_addr_2(struct file *fp, +int esc_mods_get_phys_addr_2(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS_3 *p); -int esc_mods_get_mapped_phys_addr(struct file *fp, +int esc_mods_get_mapped_phys_addr(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS *p); -int esc_mods_get_mapped_phys_addr_2(struct file *fp, +int esc_mods_get_mapped_phys_addr_2(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS_2 *p); -int esc_mods_get_mapped_phys_addr_3(struct file *fp, +int esc_mods_get_mapped_phys_addr_3(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS_3 *p); -int esc_mods_virtual_to_phys(struct file *fp, +int esc_mods_virtual_to_phys(struct mods_client *client, struct MODS_VIRTUAL_TO_PHYSICAL *p); -int esc_mods_phys_to_virtual(struct file *fp, +int esc_mods_phys_to_virtual(struct mods_client *client, struct MODS_PHYSICAL_TO_VIRTUAL *p); -int esc_mods_memory_barrier(struct file *fp); -int esc_mods_dma_map_memory(struct file *fp, +int esc_mods_memory_barrier(struct mods_client *client); +int esc_mods_dma_map_memory(struct mods_client *client, struct MODS_DMA_MAP_MEMORY *p); -int esc_mods_dma_unmap_memory(struct file *fp, +int esc_mods_dma_unmap_memory(struct mods_client *client, struct MODS_DMA_MAP_MEMORY *p); #if defined(CONFIG_PPC64) /* ppc64 */ -int esc_mods_set_ppc_tce_bypass(struct file *fp, +int esc_mods_set_ppc_tce_bypass(struct mods_client *client, struct MODS_SET_PPC_TCE_BYPASS *p); -int esc_mods_get_ats_address_range(struct file *fp, +int esc_mods_get_ats_address_range(struct mods_client *client, struct MODS_GET_ATS_ADDRESS_RANGE *p); -int esc_mods_set_nvlink_sysmem_trained(struct file *fp, +int esc_mods_set_nvlink_sysmem_trained(struct mods_client *client, struct MODS_SET_NVLINK_SYSMEM_TRAINED *p); -int esc_mods_get_nvlink_line_rate(struct file *fp, +int esc_mods_get_nvlink_line_rate(struct mods_client *client, struct MODS_GET_NVLINK_LINE_RATE *p); #endif /* acpi */ #ifdef CONFIG_ACPI -int esc_mods_eval_acpi_method(struct file *fp, +int esc_mods_eval_acpi_method(struct mods_client *client, struct MODS_EVAL_ACPI_METHOD *p); -int esc_mods_eval_dev_acpi_method(struct file *fp, +int esc_mods_eval_dev_acpi_method(struct mods_client *client, struct MODS_EVAL_DEV_ACPI_METHOD *p); -int esc_mods_eval_dev_acpi_method_2(struct file *fp, +int esc_mods_eval_dev_acpi_method_2(struct mods_client *client, struct MODS_EVAL_DEV_ACPI_METHOD_2 *p); -int esc_mods_eval_dev_acpi_method_3(struct file *fp, +int esc_mods_eval_dev_acpi_method_3(struct mods_client *client, struct MODS_EVAL_DEV_ACPI_METHOD_3 *p); -int esc_mods_acpi_get_ddc(struct file *fp, struct MODS_ACPI_GET_DDC *p); -int esc_mods_acpi_get_ddc_2(struct file *fp, struct MODS_ACPI_GET_DDC_2 *p); +int esc_mods_acpi_get_ddc(struct mods_client *client, + struct MODS_ACPI_GET_DDC *p); +int esc_mods_acpi_get_ddc_2(struct mods_client *client, + struct MODS_ACPI_GET_DDC_2 *p); #endif /* pci */ #ifdef CONFIG_PCI -int esc_mods_find_pci_dev(struct file *fp, struct MODS_FIND_PCI_DEVICE *p); -int esc_mods_find_pci_dev_2(struct file *fp, +int esc_mods_find_pci_dev(struct mods_client *client, + struct MODS_FIND_PCI_DEVICE *p); +int esc_mods_find_pci_dev_2(struct mods_client *client, struct MODS_FIND_PCI_DEVICE_2 *p); -int esc_mods_find_pci_class_code(struct file *fp, +int esc_mods_find_pci_class_code(struct mods_client *client, struct MODS_FIND_PCI_CLASS_CODE *p); -int esc_mods_find_pci_class_code_2(struct file *fp, +int esc_mods_find_pci_class_code_2(struct mods_client *client, struct MODS_FIND_PCI_CLASS_CODE_2 *p); -int esc_mods_pci_get_bar_info(struct file *fp, struct MODS_PCI_GET_BAR_INFO *p); -int esc_mods_pci_get_bar_info_2(struct file *fp, +int esc_mods_pci_get_bar_info(struct mods_client *client, + struct MODS_PCI_GET_BAR_INFO *p); +int esc_mods_pci_get_bar_info_2(struct mods_client *client, struct MODS_PCI_GET_BAR_INFO_2 *p); -int esc_mods_pci_get_irq(struct file *fp, struct MODS_PCI_GET_IRQ *p); -int esc_mods_pci_get_irq_2(struct file *fp, +int esc_mods_pci_get_irq(struct mods_client *client, + struct MODS_PCI_GET_IRQ *p); +int esc_mods_pci_get_irq_2(struct mods_client *client, struct MODS_PCI_GET_IRQ_2 *p); -int esc_mods_pci_read(struct file *fp, struct MODS_PCI_READ *p); -int esc_mods_pci_read_2(struct file *fp, struct MODS_PCI_READ_2 *p); -int esc_mods_pci_write(struct file *fp, struct MODS_PCI_WRITE *p); -int esc_mods_pci_write_2(struct file *fp, struct MODS_PCI_WRITE_2 *p); -int esc_mods_pci_bus_add_dev(struct file *fp, +int esc_mods_pci_read(struct mods_client *client, struct MODS_PCI_READ *p); +int esc_mods_pci_read_2(struct mods_client *client, struct MODS_PCI_READ_2 *p); +int esc_mods_pci_write(struct mods_client *client, struct MODS_PCI_WRITE *p); +int esc_mods_pci_write_2(struct mods_client *client, + struct MODS_PCI_WRITE_2 *p); +int esc_mods_pci_bus_add_dev(struct mods_client *client, struct MODS_PCI_BUS_ADD_DEVICES *p); -int esc_mods_pci_hot_reset(struct file *fp, +int esc_mods_pci_bus_remove_dev(struct mods_client *client, + struct MODS_PCI_BUS_REMOVE_DEV *p); +int esc_mods_pci_hot_reset(struct mods_client *client, struct MODS_PCI_HOT_RESET *p); -int esc_mods_pio_read(struct file *fp, struct MODS_PIO_READ *p); -int esc_mods_pio_write(struct file *fp, struct MODS_PIO_WRITE *p); -int esc_mods_device_numa_info(struct file *fp, +int esc_mods_pio_read(struct mods_client *client, struct MODS_PIO_READ *p); +int esc_mods_pio_write(struct mods_client *client, struct MODS_PIO_WRITE *p); +int esc_mods_device_numa_info(struct mods_client *client, struct MODS_DEVICE_NUMA_INFO *p); -int esc_mods_device_numa_info_2(struct file *fp, +int esc_mods_device_numa_info_2(struct mods_client *client, struct MODS_DEVICE_NUMA_INFO_2 *p); -int esc_mods_get_iommu_state(struct file *pfile, +int esc_mods_get_iommu_state(struct mods_client *client, struct MODS_GET_IOMMU_STATE *state); -int esc_mods_get_iommu_state_2(struct file *pfile, +int esc_mods_get_iommu_state_2(struct mods_client *client, struct MODS_GET_IOMMU_STATE *state); -int esc_mods_pci_set_dma_mask(struct file *pfile, +int esc_mods_pci_set_dma_mask(struct mods_client *client, struct MODS_PCI_DMA_MASK *dma_mask); #endif /* irq */ #if defined(MODS_TEGRA) && defined(CONFIG_OF_IRQ) && defined(CONFIG_OF) -int esc_mods_map_irq(struct file *fp, struct MODS_DT_INFO *p); +int esc_mods_map_irq(struct mods_client *client, struct MODS_DT_INFO *p); #endif -int esc_mods_register_irq(struct file *fp, struct MODS_REGISTER_IRQ *p); -int esc_mods_register_irq_2(struct file *fp, +int esc_mods_register_irq(struct mods_client *client, + struct MODS_REGISTER_IRQ *p); +int esc_mods_register_irq_2(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p); -int esc_mods_register_irq_3(struct file *fp, +int esc_mods_register_irq_3(struct mods_client *client, struct MODS_REGISTER_IRQ_3 *p); -int esc_mods_unregister_irq(struct file *fp, struct MODS_REGISTER_IRQ *p); -int esc_mods_unregister_irq_2(struct file *fp, +int esc_mods_unregister_irq(struct mods_client *client, + struct MODS_REGISTER_IRQ *p); +int esc_mods_unregister_irq_2(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p); -int esc_mods_query_irq(struct file *fp, struct MODS_QUERY_IRQ *p); -int esc_mods_query_irq_2(struct file *fp, struct MODS_QUERY_IRQ_2 *p); -int esc_mods_irq_handled(struct file *fp, struct MODS_REGISTER_IRQ *p); -int esc_mods_irq_handled_2(struct file *fp, +int esc_mods_query_irq(struct mods_client *client, struct MODS_QUERY_IRQ *p); +int esc_mods_query_irq_2(struct mods_client *client, + struct MODS_QUERY_IRQ_2 *p); +int esc_mods_irq_handled(struct mods_client *client, + struct MODS_REGISTER_IRQ *p); +int esc_mods_irq_handled_2(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p); -int esc_mods_register_irq_4(struct file *fp, +int esc_mods_register_irq_4(struct mods_client *client, struct MODS_REGISTER_IRQ_4 *p); -int esc_mods_query_irq_3(struct file *fp, struct MODS_QUERY_IRQ_3 *p); +int esc_mods_query_irq_3(struct mods_client *client, + struct MODS_QUERY_IRQ_3 *p); #ifdef MODS_TEGRA /* clock */ -int esc_mods_get_clock_handle(struct file *fp, +int esc_mods_get_clock_handle(struct mods_client *client, struct MODS_GET_CLOCK_HANDLE *p); -int esc_mods_set_clock_rate(struct file *fp, struct MODS_CLOCK_RATE *p); -int esc_mods_get_clock_rate(struct file *fp, struct MODS_CLOCK_RATE *p); -int esc_mods_get_clock_max_rate(struct file *fp, struct MODS_CLOCK_RATE *p); -int esc_mods_set_clock_max_rate(struct file *fp, struct MODS_CLOCK_RATE *p); -int esc_mods_set_clock_parent(struct file *fp, struct MODS_CLOCK_PARENT *p); -int esc_mods_get_clock_parent(struct file *fp, struct MODS_CLOCK_PARENT *p); -int esc_mods_enable_clock(struct file *fp, struct MODS_CLOCK_HANDLE *p); -int esc_mods_disable_clock(struct file *fp, struct MODS_CLOCK_HANDLE *p); -int esc_mods_is_clock_enabled(struct file *fp, +int esc_mods_set_clock_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p); +int esc_mods_get_clock_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p); +int esc_mods_get_clock_max_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p); +int esc_mods_set_clock_max_rate(struct mods_client *client, + struct MODS_CLOCK_RATE *p); +int esc_mods_set_clock_parent(struct mods_client *client, + struct MODS_CLOCK_PARENT *p); +int esc_mods_get_clock_parent(struct mods_client *client, + struct MODS_CLOCK_PARENT *p); +int esc_mods_enable_clock(struct mods_client *client, + struct MODS_CLOCK_HANDLE *p); +int esc_mods_disable_clock(struct mods_client *client, + struct MODS_CLOCK_HANDLE *p); +int esc_mods_is_clock_enabled(struct mods_client *client, struct MODS_CLOCK_ENABLED *p); -int esc_mods_clock_reset_assert(struct file *fp, +int esc_mods_clock_reset_assert(struct mods_client *client, struct MODS_CLOCK_HANDLE *p); -int esc_mods_clock_reset_deassert(struct file *fp, +int esc_mods_clock_reset_deassert(struct mods_client *client, struct MODS_CLOCK_HANDLE *p); -int esc_mods_flush_cpu_cache_range(struct file *fp, +int esc_mods_flush_cpu_cache_range(struct mods_client *client, struct MODS_FLUSH_CPU_CACHE_RANGE *p); -int esc_mods_dma_alloc_coherent(struct file *fp, +int esc_mods_dma_alloc_coherent(struct mods_client *client, struct MODS_DMA_COHERENT_MEM_HANDLE *p); -int esc_mods_dma_free_coherent(struct file *fp, +int esc_mods_dma_free_coherent(struct mods_client *client, struct MODS_DMA_COHERENT_MEM_HANDLE *p); -int esc_mods_dma_copy_to_user(struct file *fp, +int esc_mods_dma_copy_to_user(struct mods_client *client, struct MODS_DMA_COPY_TO_USER *p); #ifdef CONFIG_DMA_ENGINE -int esc_mods_dma_request_channel(struct file *fp, struct MODS_DMA_HANDLE *p); -int esc_mods_dma_release_channel(struct file *fp, struct MODS_DMA_HANDLE *p); -int esc_mods_dma_set_config(struct file *fp, struct MODS_DMA_CHANNEL_CONFIG *p); -int esc_mods_dma_wait(struct file *fp, struct MODS_DMA_WAIT_DESC *p); -int esc_mods_dma_submit_request(struct file *fp, +int esc_mods_dma_request_channel(struct mods_client *client, + struct MODS_DMA_HANDLE *p); +int esc_mods_dma_release_channel(struct mods_client *client, + struct MODS_DMA_HANDLE *p); +int esc_mods_dma_set_config(struct mods_client *client, + struct MODS_DMA_CHANNEL_CONFIG *p); +int esc_mods_dma_wait(struct mods_client *client, struct MODS_DMA_WAIT_DESC *p); +int esc_mods_dma_submit_request(struct mods_client *client, struct MODS_DMA_TX_DESC *p); -int esc_mods_dma_async_issue_pending(struct file *fp, - struct MODS_DMA_HANDLE *p); +int esc_mods_dma_async_issue_pending(struct mods_client *client, + struct MODS_DMA_HANDLE *p); #endif #ifdef CONFIG_TEGRA_DC -int esc_mods_tegra_dc_config_possible(struct file *fp, - struct MODS_TEGRA_DC_CONFIG_POSSIBLE *p); +int esc_mods_tegra_dc_config_possible(struct mods_client *client, + struct MODS_TEGRA_DC_CONFIG_POSSIBLE *p); #endif #ifdef MODS_HAS_NET -int esc_mods_net_force_link(struct file *fp, struct MODS_NET_DEVICE_NAME *p); +int esc_mods_net_force_link(struct mods_client *client, + struct MODS_NET_DEVICE_NAME *p); #endif #ifdef MODS_HAS_DMABUF -int esc_mods_dmabuf_get_phys_addr(struct file *fp, +int esc_mods_dmabuf_get_phys_addr(struct mods_client *client, struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *p); #else -static inline int esc_mods_dmabuf_get_phys_addr(struct file *fp, +static inline int esc_mods_dmabuf_get_phys_addr(struct mods_client *client, struct MODS_DMABUF_GET_PHYSICAL_ADDRESS *p) { return -EINVAL; } #endif #ifdef CONFIG_TEGRA_NVADSP -int esc_mods_adsp_load(struct file *fp); -int esc_mods_adsp_start(struct file *fp); -int esc_mods_adsp_stop(struct file *fp); -int esc_mods_adsp_run_app(struct file *fp, struct MODS_ADSP_RUN_APP_INFO *p); +int esc_mods_adsp_load(struct mods_client *client); +int esc_mods_adsp_start(struct mods_client *client); +int esc_mods_adsp_stop(struct mods_client *client); +int esc_mods_adsp_run_app(struct mods_client *client, + struct MODS_ADSP_RUN_APP_INFO *p); #endif /* prod */ int mods_tegra_prod_init(const struct miscdevice *misc_dev); -int esc_mods_tegra_prod_iterate_dt(struct file *fp, +int esc_mods_tegra_prod_iterate_dt(struct mods_client *client, struct MODS_TEGRA_PROD_ITERATOR *iterator); -int esc_mods_tegra_prod_is_supported(struct file *fp, +int esc_mods_tegra_prod_is_supported(struct mods_client *client, struct MODS_TEGRA_PROD_IS_SUPPORTED *tuple); -int esc_mods_tegra_prod_set_prod_all(struct file *fp, +int esc_mods_tegra_prod_set_prod_all(struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple); -int esc_mods_tegra_prod_set_prod_boot(struct file *fp, +int esc_mods_tegra_prod_set_prod_boot(struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple); -int esc_mods_tegra_prod_set_prod_by_name(struct file *fp, +int esc_mods_tegra_prod_set_prod_by_name(struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple); -int esc_mods_tegra_prod_set_prod_exact(struct file *fp, +int esc_mods_tegra_prod_set_prod_exact(struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple); #endif diff --git a/drivers/misc/mods/mods_irq.c b/drivers/misc/mods/mods_irq.c index 6435c79c..de313278 100644 --- a/drivers/misc/mods/mods_irq.c +++ b/drivers/misc/mods/mods_irq.c @@ -2,7 +2,7 @@ /* * mods_irq.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -56,8 +56,9 @@ struct mutex *mods_get_irq_mutex(void) } #ifdef CONFIG_PCI -struct en_dev_entry *mods_enable_device(struct mods_client *client, - struct pci_dev *dev) +int mods_enable_device(struct mods_client *client, + struct pci_dev *dev, + struct en_dev_entry **dev_entry) { int err = OK; struct en_dev_entry *dpriv = client->enabled_devices; @@ -66,8 +67,11 @@ struct en_dev_entry *mods_enable_device(struct mods_client *client, dpriv = pci_get_drvdata(dev); if (unlikely(dpriv)) { - if (dpriv->client_id == client->client_id) - return dpriv; + if (dpriv->client_id == client->client_id) { + if (dev_entry) + *dev_entry = dpriv; + return OK; + } mods_error_printk("invalid client %u for device %04x:%02x:%02x.%x\n", client->client_id, @@ -75,12 +79,13 @@ struct en_dev_entry *mods_enable_device(struct mods_client *client, dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); - return 0; + return -EBUSY; } dpriv = kzalloc(sizeof(*dpriv), GFP_KERNEL | __GFP_NORETRY); if (unlikely(!dpriv)) - return 0; + return -ENOMEM; + atomic_inc(&client->num_allocs); err = pci_enable_device(dev); if (unlikely(err)) { @@ -90,7 +95,8 @@ struct en_dev_entry *mods_enable_device(struct mods_client *client, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); kfree(dpriv); - return 0; + atomic_dec(&client->num_allocs); + return err; } dpriv->client_id = client->client_id; @@ -99,7 +105,9 @@ struct en_dev_entry *mods_enable_device(struct mods_client *client, client->enabled_devices = dpriv; pci_set_drvdata(dev, dpriv); - return dpriv; + if (dev_entry) + *dev_entry = dpriv; + return OK; } void mods_disable_device(struct pci_dev *dev) @@ -200,21 +208,26 @@ static const char *mods_irq_type_name(u8 irq_type) } #endif +static struct mods_client *client_from_id(u8 client_id) +{ + return &mp.clients[client_id - 1]; +} + static void wake_up_client(struct dev_irq_map *t) { - struct mods_client *client = &mp.clients[t->client_id - 1]; + struct mods_client *client = client_from_id(t->client_id); if (client) wake_up_interruptible(&client->interrupt_event); } static int rec_irq_done(struct dev_irq_map *t, - unsigned int irq_time) + unsigned int irq_time) { struct irq_q_info *q; /* Get interrupt queue */ - q = &mp.clients[t->client_id - 1].irq_queue; + q = &client_from_id(t->client_id)->irq_queue; /* Don't do anything if the IRQ has already been recorded */ if (q->head != q->tail) { @@ -266,11 +279,7 @@ static int rec_irq_done(struct dev_irq_map *t, } /* mods_irq_handle - interrupt function */ -static irqreturn_t mods_irq_handle(int irq, void *data -#ifndef MODS_IRQ_HANDLE_NO_REGS - , struct pt_regs *regs -#endif -) +static irqreturn_t mods_irq_handle(int irq, void *data) { struct dev_irq_map *t = (struct dev_irq_map *)data; int serviced = false; @@ -285,7 +294,7 @@ static irqreturn_t mods_irq_handle(int irq, void *data unsigned long flags = 0; int recorded = false; unsigned int irq_time = get_cur_time(); - struct mods_client *client = &mp.clients[t->client_id - 1]; + struct mods_client *client = client_from_id(t->client_id); spin_lock_irqsave(&client->irq_lock, flags); @@ -319,25 +328,25 @@ static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq) LOG_ENT(); - for (client_idx = 0; client_idx < MODS_MAX_CLIENTS; client_idx++) { + for (client_idx = 1; client_idx <= MODS_MAX_CLIENTS; client_idx++) { struct dev_irq_map *t = NULL; struct dev_irq_map *next = NULL; list_for_each_entry_safe(t, next, - &mp.clients[client_idx].irq_list, + &client_from_id(client_idx)->irq_list, list) { if (t->apic_irq == irq) { if (client_id == 0) { ret = IRQ_FOUND; } else { - ret = (client_id == client_idx + 1) - ? IRQ_FOUND : IRQ_NOT_FOUND; + ret = (client_id == client_idx) + ? IRQ_FOUND : IRQ_NOT_FOUND; } /* Break out of the outer loop */ - client_idx = MODS_MAX_CLIENTS; + client_idx = MODS_MAX_CLIENTS - 1; break; } } @@ -396,7 +405,7 @@ static void setup_mask_info(struct dev_irq_map *newmap, } #endif -static int add_irq_map(u8 client_id, +static int add_irq_map(struct mods_client *client, struct pci_dev *dev, struct MODS_REGISTER_IRQ_4 *p, u32 irq, @@ -413,11 +422,12 @@ static int add_irq_map(u8 client_id, LOG_EXT(); return -ENOMEM; } + atomic_inc(&client->num_allocs); /* Fill out the new entry */ newmap->apic_irq = irq; newmap->dev = dev; - newmap->client_id = client_id; + newmap->client_id = client->client_id; newmap->dev_irq_aperture = 0; newmap->mask_info_cnt = 0; newmap->type = irq_type; @@ -432,12 +442,13 @@ static int add_irq_map(u8 client_id, newmap)) { mods_error_printk("unable to enable IRQ 0x%x\n", irq); kfree(newmap); + atomic_dec(&client->num_allocs); LOG_EXT(); return -EPERM; } /* Add the new entry to the list of all registered interrupts */ - list_add(&newmap->list, &mp.clients[client_id - 1].irq_list); + list_add(&newmap->list, &client->irq_list); #ifdef CONFIG_PCI /* Map BAR0 to be able to disable interrupts */ @@ -520,7 +531,7 @@ static int add_irq_map(u8 client_id, static void mods_free_map(struct dev_irq_map *del) { unsigned long flags = 0; - struct mods_client *client = &mp.clients[del->client_id - 1]; + struct mods_client *client = client_from_id(del->client_id); LOG_ENT(); @@ -542,6 +553,7 @@ static void mods_free_map(struct dev_irq_map *del) /* Free memory */ kfree(del); + atomic_dec(&client->num_allocs); LOG_EXT(); } @@ -563,7 +575,7 @@ void mods_cleanup_irq(void) LOG_ENT(); for (i = 0; i < MODS_MAX_CLIENTS; i++) { - if (mp.client_flags && (1 << i)) + if (mp.client_flags & (1 << i)) mods_free_client(i + 1); } LOG_EXT(); @@ -571,7 +583,7 @@ void mods_cleanup_irq(void) int mods_irq_event_check(u8 client_id) { - struct irq_q_info *q = &mp.clients[client_id - 1].irq_queue; + struct irq_q_info *q = &client_from_id(client_id)->irq_queue; unsigned int pos = (1 << (client_id - 1)); if (!(mp.client_flags & pos)) @@ -585,7 +597,7 @@ int mods_irq_event_check(u8 client_id) struct mods_client *mods_alloc_client(void) { - u8 idx = 0; + u8 idx; u8 max_clients = 1; LOG_ENT(); @@ -594,17 +606,17 @@ struct mods_client *mods_alloc_client(void) (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE)) max_clients = MODS_MAX_CLIENTS; - for (idx = 0; idx < max_clients; idx++) { - if (!test_and_set_bit(idx, &mp.client_flags)) { - struct mods_client *client = &mp.clients[idx]; + for (idx = 1; idx <= max_clients; idx++) { + if (!test_and_set_bit(idx - 1, &mp.client_flags)) { + struct mods_client *client = client_from_id(idx); mods_debug_printk(DEBUG_IOCTL, "open client %u (bit mask 0x%lx)\n", - (unsigned int)(idx + 1), + (unsigned int)idx, mp.client_flags); memset(client, 0, sizeof(*client)); - client->client_id = idx + 1; + client->client_id = idx; client->access_token = MODS_ACCESS_TOKEN_NONE; mutex_init(&client->mtx); spin_lock_init(&client->irq_lock); @@ -612,6 +624,7 @@ struct mods_client *mods_alloc_client(void) INIT_LIST_HEAD(&client->irq_list); INIT_LIST_HEAD(&client->mem_alloc_list); INIT_LIST_HEAD(&client->mem_map_list); + INIT_LIST_HEAD(&client->free_mem_list); #if defined(CONFIG_PPC64) INIT_LIST_HEAD(&client->ppc_tce_bypass_list); INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list); @@ -629,7 +642,7 @@ struct mods_client *mods_alloc_client(void) static int mods_free_irqs(u8 client_id, struct pci_dev *dev) { #ifdef CONFIG_PCI - struct mods_client *client = &mp.clients[client_id - 1]; + struct mods_client *client = client_from_id(client_id); struct dev_irq_map *del = NULL; struct dev_irq_map *next; struct en_dev_entry *dpriv; @@ -650,9 +663,9 @@ static int mods_free_irqs(u8 client_id, struct pci_dev *dev) return OK; } - if (dpriv->client_id != client_id) { + if (dpriv->client_id != client->client_id) { mods_error_printk("invalid client %u for device %04x:%02x:%02x.%x\n", - client_id, + client->client_id, pci_domain_nr(dev->bus), dev->bus->number, PCI_SLOT(dev->devfn), @@ -701,6 +714,8 @@ static int mods_free_irqs(u8 client_id, struct pci_dev *dev) if (irq_type == MODS_IRQ_TYPE_MSIX) { pci_disable_msix(dev); kfree(dpriv->msix_entries); + if (dpriv->msix_entries) + atomic_dec(&client->num_allocs); dpriv->msix_entries = 0; } else if (irq_type == MODS_IRQ_TYPE_MSI) { pci_disable_msi(dev); @@ -733,7 +748,7 @@ void mods_free_client_interrupts(struct mods_client *client) void mods_free_client(u8 client_id) { - struct mods_client *client = &mp.clients[client_id - 1]; + struct mods_client *client = client_from_id(client_id); LOG_ENT(); @@ -748,12 +763,14 @@ void mods_free_client(u8 client_id) } #ifdef CONFIG_PCI -static int mods_allocate_irqs(u8 client_id, struct file *pfile, - struct pci_dev *dev, u32 nvecs, u32 flags) +static int mods_allocate_irqs(struct mods_client *client, + struct pci_dev *dev, + u32 nvecs, + u32 flags) { - struct mods_client *client = pfile->private_data; struct en_dev_entry *dpriv; unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(flags); + int err = OK; LOG_ENT(); @@ -776,10 +793,11 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); LOG_EXT(); - return -EINVAL; + return -ENOENT; } #else mods_error_printk("the kernel does not support MSI!\n"); + LOG_EXT(); return -EINVAL; #endif } else if (irq_type == MODS_IRQ_TYPE_MSIX) { @@ -791,25 +809,26 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); LOG_EXT(); - return -EINVAL; + return -ENOENT; } #else mods_error_printk("the kernel does not support MSI-X!\n"); + LOG_EXT(); return -EINVAL; #endif } /* Enable device on the PCI bus */ - dpriv = mods_enable_device(client, dev); - if (!dpriv) { + err = mods_enable_device(client, dev, &dpriv); + if (err) { LOG_EXT(); - return -EINVAL; + return err; } if (irq_type == MODS_IRQ_TYPE_INT) { /* use legacy irq */ if (nvecs != 1) { - mods_error_printk("INTA: only 1 INTA vector supported requested %d!\n", + mods_error_printk("INTA: only 1 INTA vector supported, requested %u!\n", nvecs); LOG_EXT(); return -EINVAL; @@ -820,19 +839,20 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, #ifdef CONFIG_PCI_MSI else if (irq_type == MODS_IRQ_TYPE_MSI) { if (nvecs != 1) { - mods_error_printk("MSI: only 1 MSI vector supported requested %d!\n", + mods_error_printk("MSI: only 1 MSI vector supported, requested %u!\n", nvecs); LOG_EXT(); return -EINVAL; } - if (pci_enable_msi(dev) != 0) { + err = pci_enable_msi(dev); + if (err) { mods_error_printk("unable to enable MSI on dev %04x:%02x:%02x.%x\n", pci_domain_nr(dev->bus), dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); LOG_EXT(); - return -EINVAL; + return err; } dpriv->nvecs = 1; } else if (irq_type == MODS_IRQ_TYPE_MSIX) { @@ -848,6 +868,7 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, LOG_EXT(); return -ENOMEM; } + atomic_inc(&client->num_allocs); for (i = 0; i < nvecs; i++) entries[i].entry = (uint16_t)i; @@ -863,6 +884,7 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, "could not allocate the requested number of MSI-X vectors=%d return=%d!\n", nvecs, cnt); kfree(entries); + atomic_dec(&client->num_allocs); LOG_EXT(); return cnt; } @@ -879,6 +901,7 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, "could not allocate the requested number of MSI-X vectors=%d return=%d!\n", nvecs, cnt); kfree(entries); + atomic_dec(&client->num_allocs); LOG_EXT(); if (cnt > 0) cnt = -ENOSPC; @@ -909,32 +932,23 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, return -EINVAL; } - dpriv->client_id = client_id; + dpriv->client_id = client->client_id; dpriv->irq_flags = flags; LOG_EXT(); return OK; } -static int mods_register_pci_irq(struct file *pfile, +static int mods_register_pci_irq(struct mods_client *client, struct MODS_REGISTER_IRQ_4 *p) { int err = OK; unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); struct pci_dev *dev; struct en_dev_entry *dpriv; - u8 client_id; int i; LOG_ENT(); - /* Identify the caller */ - client_id = get_client_id(pfile); - if (unlikely(!is_client_id_valid(client_id))) { - WARN_ON(!is_client_id_valid(client_id)); - LOG_EXT(); - return -EINVAL; - } - if (unlikely(!p->irq_count)) { mods_error_printk("no irq's requested!\n"); LOG_EXT(); @@ -942,7 +956,7 @@ static int mods_register_pci_irq(struct file *pfile, } /* Get the PCI device structure for the specified device from kernel */ - err = mods_find_pci_dev(pfile, &p->dev, &dev); + err = mods_find_pci_dev(client, &p->dev, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -962,7 +976,7 @@ static int mods_register_pci_irq(struct file *pfile, dpriv = pci_get_drvdata(dev); if (dpriv) { - if (dpriv->client_id != client_id) { + if (dpriv->client_id != client->client_id) { mods_error_printk("dev %04x:%02x:%02x.%x already owned by client %u\n", p->dev.domain, p->dev.bus, @@ -972,7 +986,7 @@ static int mods_register_pci_irq(struct file *pfile, mutex_unlock(&mp.mtx); pci_dev_put(dev); LOG_EXT(); - return -EINVAL; + return -EBUSY; } if (dpriv->nvecs) { mods_error_printk("interrupt for dev %04x:%02x:%02x.%x already registered\n", @@ -987,14 +1001,15 @@ static int mods_register_pci_irq(struct file *pfile, } } - if (mods_allocate_irqs(client_id, pfile, dev, p->irq_count, - p->irq_flags)) { + err = mods_allocate_irqs(client, dev, p->irq_count, + p->irq_flags); + if (err) { mods_error_printk("could not allocate irqs for irq_type %d\n", irq_type); mutex_unlock(&mp.mtx); pci_dev_put(dev); LOG_EXT(); - return -EINVAL; + return err; } dpriv = pci_get_drvdata(dev); @@ -1004,7 +1019,7 @@ static int mods_register_pci_irq(struct file *pfile, (irq_type == MODS_IRQ_TYPE_MSI)) ? dev->irq : dpriv->msix_entries[i].vector; - err = add_irq_map(client_id, dev, p, irq, i); + err = add_irq_map(client, dev, p, irq, i); if (unlikely(err)) { #ifdef CONFIG_PCI_MSI if (irq_type == MODS_IRQ_TYPE_MSI) @@ -1023,23 +1038,14 @@ static int mods_register_pci_irq(struct file *pfile, } #endif /* CONFIG_PCI */ -static int mods_register_cpu_irq(struct file *pfile, +static int mods_register_cpu_irq(struct mods_client *client, struct MODS_REGISTER_IRQ_4 *p) { - u8 client_id; u32 irq = p->dev.bus; int err; LOG_ENT(); - /* Identify the caller */ - client_id = get_client_id(pfile); - if (unlikely(!is_client_id_valid(client_id))) { - WARN_ON(!is_client_id_valid(client_id)); - LOG_EXT(); - return -EINVAL; - } - if (unlikely(mutex_lock_interruptible(&mp.mtx))) { LOG_EXT(); return -EINTR; @@ -1055,7 +1061,7 @@ static int mods_register_cpu_irq(struct file *pfile, } /* Register interrupt */ - err = add_irq_map(client_id, 0, p, irq, 0); + err = add_irq_map(client, 0, p, irq, 0); mutex_unlock(&mp.mtx); LOG_EXT(); @@ -1063,31 +1069,22 @@ static int mods_register_cpu_irq(struct file *pfile, } #ifdef CONFIG_PCI -static int mods_unregister_pci_irq(struct file *pfile, +static int mods_unregister_pci_irq(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p) { struct pci_dev *dev; - u8 client_id; int err = OK; LOG_ENT(); - /* Identify the caller */ - client_id = get_client_id(pfile); - if (unlikely(!is_client_id_valid(client_id))) { - WARN_ON(!is_client_id_valid(client_id)); - LOG_EXT(); - return -EINVAL; - } - /* Get the PCI device structure for the specified device from kernel */ - err = mods_find_pci_dev(pfile, &p->dev, &dev); + err = mods_find_pci_dev(client, &p->dev, &dev); if (unlikely(err)) { LOG_EXT(); return err; } - err = mods_free_irqs(client_id, dev); + err = mods_free_irqs(client->client_id, dev); pci_dev_put(dev); LOG_EXT(); @@ -1095,35 +1092,24 @@ static int mods_unregister_pci_irq(struct file *pfile, } #endif -static int mods_unregister_cpu_irq(struct file *pfile, +static int mods_unregister_cpu_irq(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p) { struct dev_irq_map *del = NULL; struct dev_irq_map *next; unsigned int irq; - u8 client_id; - struct mods_client *client; LOG_ENT(); irq = p->dev.bus; - /* Identify the caller */ - client_id = get_client_id(pfile); - if (unlikely(!is_client_id_valid(client_id))) { - WARN_ON(!is_client_id_valid(client_id)); - LOG_EXT(); - return -EINVAL; - } - client = &mp.clients[client_id = 1]; - if (unlikely(mutex_lock_interruptible(&mp.mtx))) { LOG_EXT(); return -EINTR; } /* Determine if the interrupt is already hooked by this client */ - if (mods_lookup_cpu_irq(client_id, irq) == IRQ_NOT_FOUND) { + if (mods_lookup_cpu_irq(client->client_id, irq) == IRQ_NOT_FOUND) { mods_error_printk( "IRQ 0x%x not hooked, can't unhook\n", irq); @@ -1159,22 +1145,22 @@ static int mods_unregister_cpu_irq(struct file *pfile, * ESCAPE CALL FUNCTIONS * *************************/ -int esc_mods_register_irq_4(struct file *pfile, +int esc_mods_register_irq_4(struct mods_client *client, struct MODS_REGISTER_IRQ_4 *p) { u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); if (irq_type == MODS_IRQ_TYPE_CPU) - return mods_register_cpu_irq(pfile, p); + return mods_register_cpu_irq(client, p); #ifdef CONFIG_PCI - return mods_register_pci_irq(pfile, p); + return mods_register_pci_irq(client, p); #else mods_error_printk("PCI not available\n"); return -EINVAL; #endif } -int esc_mods_register_irq_3(struct file *pfile, +int esc_mods_register_irq_3(struct mods_client *client, struct MODS_REGISTER_IRQ_3 *p) { struct MODS_REGISTER_IRQ_4 irq_data = { {0} }; @@ -1202,10 +1188,10 @@ int esc_mods_register_irq_3(struct file *pfile, irq_data.irq_count = 1; irq_data.irq_flags = p->irq_type; - return esc_mods_register_irq_4(pfile, &irq_data); + return esc_mods_register_irq_4(client, &irq_data); } -int esc_mods_register_irq_2(struct file *pfile, +int esc_mods_register_irq_2(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p) { struct MODS_REGISTER_IRQ_4 irq_data = { {0} }; @@ -1220,7 +1206,7 @@ int esc_mods_register_irq_2(struct file *pfile, struct pci_dev *dev; int err; - err = mods_find_pci_dev(pfile, &p->dev, &dev); + err = mods_find_pci_dev(client, &p->dev, &dev); if (unlikely(err)) return err; @@ -1231,10 +1217,10 @@ int esc_mods_register_irq_2(struct file *pfile, } #endif - return esc_mods_register_irq_4(pfile, &irq_data); + return esc_mods_register_irq_4(client, &irq_data); } -int esc_mods_register_irq(struct file *pfile, +int esc_mods_register_irq(struct mods_client *client, struct MODS_REGISTER_IRQ *p) { struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; @@ -1245,39 +1231,38 @@ int esc_mods_register_irq(struct file *pfile, register_irq.dev.function = p->dev.function; register_irq.type = p->type; - return esc_mods_register_irq_2(pfile, ®ister_irq); + return esc_mods_register_irq_2(client, ®ister_irq); } -int esc_mods_unregister_irq_2(struct file *pfile, +int esc_mods_unregister_irq_2(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p) { if (p->type == MODS_IRQ_TYPE_CPU) - return mods_unregister_cpu_irq(pfile, p); + return mods_unregister_cpu_irq(client, p); #ifdef CONFIG_PCI - return mods_unregister_pci_irq(pfile, p); + return mods_unregister_pci_irq(client, p); #else return -EINVAL; #endif } -int esc_mods_unregister_irq(struct file *pfile, +int esc_mods_unregister_irq(struct mods_client *client, struct MODS_REGISTER_IRQ *p) { struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; - register_irq.dev.domain = 0; - register_irq.dev.bus = p->dev.bus; - register_irq.dev.device = p->dev.device; - register_irq.dev.function = p->dev.function; - register_irq.type = p->type; + register_irq.dev.domain = 0; + register_irq.dev.bus = p->dev.bus; + register_irq.dev.device = p->dev.device; + register_irq.dev.function = p->dev.function; + register_irq.type = p->type; - return esc_mods_unregister_irq_2(pfile, ®ister_irq); + return esc_mods_unregister_irq_2(client, ®ister_irq); } -int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) +int esc_mods_query_irq_3(struct mods_client *client, + struct MODS_QUERY_IRQ_3 *p) { - u8 client_id; - struct mods_client *client; struct irq_q_info *q = NULL; unsigned int i = 0; unsigned long flags = 0; @@ -1285,15 +1270,6 @@ int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) LOG_ENT(); - /* Identify the caller */ - client_id = get_client_id(pfile); - if (unlikely(!is_client_id_valid(client_id))) { - WARN_ON(!is_client_id_valid(client_id)); - LOG_EXT(); - return -EINVAL; - } - client = &mp.clients[client_id - 1]; - /* Clear return array */ memset(p->irq_list, 0xFF, sizeof(p->irq_list)); @@ -1354,12 +1330,13 @@ int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) return OK; } -int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) +int esc_mods_query_irq_2(struct mods_client *client, + struct MODS_QUERY_IRQ_2 *p) { int retval, i; struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; - retval = esc_mods_query_irq_3(pfile, &query_irq); + retval = esc_mods_query_irq_3(client, &query_irq); if (retval) return retval; @@ -1371,13 +1348,13 @@ int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) return OK; } -int esc_mods_query_irq(struct file *pfile, +int esc_mods_query_irq(struct mods_client *client, struct MODS_QUERY_IRQ *p) { int retval, i; struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; - retval = esc_mods_query_irq_3(pfile, &query_irq); + retval = esc_mods_query_irq_3(client, &query_irq); if (retval) return retval; @@ -1392,11 +1369,9 @@ int esc_mods_query_irq(struct file *pfile, return OK; } -int esc_mods_irq_handled_2(struct file *pfile, +int esc_mods_irq_handled_2(struct mods_client *client, struct MODS_REGISTER_IRQ_2 *p) { - u8 client_id; - struct mods_client *client; unsigned long flags = 0; u32 irq = p->dev.bus; struct dev_irq_map *t = NULL; @@ -1408,15 +1383,6 @@ int esc_mods_irq_handled_2(struct file *pfile, LOG_ENT(); - /* Identify the caller */ - client_id = get_client_id(pfile); - if (unlikely(!is_client_id_valid(client_id))) { - WARN_ON(!is_client_id_valid(client_id)); - LOG_EXT(); - return -EINVAL; - } - client = &mp.clients[client_id - 1]; - /* Print info */ mods_debug_printk(DEBUG_ISR_DETAILED, "mark CPU IRQ 0x%x handled\n", irq); @@ -1444,22 +1410,22 @@ int esc_mods_irq_handled_2(struct file *pfile, return err; } -int esc_mods_irq_handled(struct file *pfile, +int esc_mods_irq_handled(struct mods_client *client, struct MODS_REGISTER_IRQ *p) { struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; - register_irq.dev.domain = 0; - register_irq.dev.bus = p->dev.bus; - register_irq.dev.device = p->dev.device; - register_irq.dev.function = p->dev.function; - register_irq.type = p->type; + register_irq.dev.domain = 0; + register_irq.dev.bus = p->dev.bus; + register_irq.dev.device = p->dev.device; + register_irq.dev.function = p->dev.function; + register_irq.type = p->type; - return esc_mods_irq_handled_2(pfile, ®ister_irq); + return esc_mods_irq_handled_2(client, ®ister_irq); } #if defined(MODS_TEGRA) && defined(CONFIG_OF_IRQ) && defined(CONFIG_OF) -int esc_mods_map_irq(struct file *pfile, +int esc_mods_map_irq(struct mods_client *client, struct MODS_DT_INFO *p) { int err; diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c index 1b446cec..723f844b 100644 --- a/drivers/misc/mods/mods_krnl.c +++ b/drivers/misc/mods/mods_krnl.c @@ -36,7 +36,7 @@ # include # include #endif -#ifdef MODS_HAS_MSR +#ifdef CONFIG_X86 # include #endif @@ -55,14 +55,14 @@ static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs); /* character driver entry points */ static const struct file_operations mods_fops = { - .owner = THIS_MODULE, - .open = mods_krnl_open, - .release = mods_krnl_close, - .poll = mods_krnl_poll, - .mmap = mods_krnl_mmap, + .owner = THIS_MODULE, + .open = mods_krnl_open, + .release = mods_krnl_close, + .poll = mods_krnl_poll, + .mmap = mods_krnl_mmap, .unlocked_ioctl = mods_krnl_ioctl, #if defined(HAVE_COMPAT_IOCTL) - .compat_ioctl = mods_krnl_ioctl, + .compat_ioctl = mods_krnl_ioctl, #endif }; @@ -70,11 +70,11 @@ static const struct file_operations mods_fops = { struct miscdevice mods_dev = { .minor = MISC_DYNAMIC_MINOR, - .name = DEVICE_NAME, - .fops = &mods_fops + .name = DEVICE_NAME, + .fops = &mods_fops }; -#if defined(MODS_CAN_REGISTER_PCI_DEV) +#if defined(CONFIG_PCI) static pci_ers_result_t mods_pci_error_detected(struct pci_dev *, enum pci_channel_state); static pci_ers_result_t mods_pci_mmio_enabled(struct pci_dev *); @@ -162,7 +162,8 @@ static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs) return numvfs; } -static int esc_mods_set_num_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) +static int esc_mods_set_num_vf(struct mods_client *client, + struct MODS_SET_NUM_VF *p) { int err; struct pci_dev *dev = NULL; @@ -170,7 +171,7 @@ static int esc_mods_set_num_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) LOG_ENT(); /* Get the PCI device structure for the specified device from kernel */ - err = mods_find_pci_dev(pfile, &p->dev, &dev); + err = mods_find_pci_dev(client, &p->dev, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -189,7 +190,8 @@ static int esc_mods_set_num_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) return err; } -static int esc_mods_set_total_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) +static int esc_mods_set_total_vf(struct mods_client *client, + struct MODS_SET_NUM_VF *p) { int err; struct pci_dev *dev = NULL; @@ -200,7 +202,7 @@ static int esc_mods_set_total_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) "pci_sriov_set_totalvfs(totalvfs=%d)\n", p->numvfs); /* Get the PCI device structure for the specified device from kernel */ - err = mods_find_pci_dev(pfile, &p->dev, &dev); + err = mods_find_pci_dev(client, &p->dev, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -267,6 +269,21 @@ u32 mods_get_access_token(void) return access_token; } +static int validate_client(struct mods_client *client) +{ + if (!client) { + mods_error_printk("invalid client\n"); + return false; + } + + if (client->client_id < 1 || client->client_id > MODS_MAX_CLIENTS) { + mods_error_printk("invalid client id %u\n", client->client_id); + return false; + } + + return true; +} + static int mods_set_access_token(u32 tok) { /* When setting a null token, the existing token must match the @@ -282,10 +299,8 @@ static int mods_set_access_token(u32 tok) return OK; } -static int mods_check_access_token(struct file *fp) +static int mods_check_access_token(struct mods_client *client) { - struct mods_client *client = fp->private_data; - if (client->access_token != mods_get_access_token()) return -EFAULT; @@ -307,7 +322,7 @@ static int __init mods_init_module(void) if (rc < 0) return -EBUSY; -#if defined(MODS_CAN_REGISTER_PCI_DEV) +#if defined(CONFIG_PCI) rc = pci_register_driver(&mods_pci_driver); if (rc < 0) return -EBUSY; @@ -352,7 +367,7 @@ static void __exit mods_exit_module(void) mods_cleanup_irq(); -#if defined(MODS_CAN_REGISTER_PCI_DEV) +#if defined(CONFIG_PCI) pci_unregister_driver(&mods_pci_driver); #endif @@ -408,6 +423,7 @@ static void mods_disable_all_devices(struct mods_client *client) mods_disable_device(old->dev); client->enabled_devices = old->next; kfree(old); + atomic_dec(&client->num_allocs); } mutex_unlock(mods_get_irq_mutex()); @@ -421,20 +437,19 @@ static void mods_disable_all_devices(struct mods_client *client) #endif } -static int mods_resume_console(struct file *pfile); +static int mods_resume_console(struct mods_client *client); /********************* * MAPPING FUNCTIONS * *********************/ static int mods_register_mapping( - struct file *fp, + struct mods_client *client, struct MODS_MEM_INFO *p_mem_info, u64 dma_addr, u64 virtual_address, u64 mapping_length) { struct SYS_MAP_MEMORY *p_map_mem; - struct mods_client *client = fp->private_data; LOG_ENT(); @@ -443,6 +458,7 @@ static int mods_register_mapping( LOG_EXT(); return -ENOMEM; } + atomic_inc(&client->num_allocs); p_map_mem->dma_addr = dma_addr; p_map_mem->virtual_addr = virtual_address; @@ -459,10 +475,10 @@ static int mods_register_mapping( return OK; } -static void mods_unregister_mapping(struct file *fp, u64 virtual_address) +static void mods_unregister_mapping(struct mods_client *client, + u64 virtual_address) { struct SYS_MAP_MEMORY *p_map_mem; - struct mods_client *client = fp->private_data; struct list_head *head = &client->mem_map_list; struct list_head *iter; @@ -477,6 +493,7 @@ static void mods_unregister_mapping(struct file *fp, u64 virtual_address) /* free our data struct which keeps track of mapping */ kfree(p_map_mem); + atomic_dec(&client->num_allocs); return; } @@ -484,10 +501,9 @@ static void mods_unregister_mapping(struct file *fp, u64 virtual_address) LOG_EXT(); } -static void mods_unregister_all_mappings(struct file *fp) +static void mods_unregister_all_mappings(struct mods_client *client) { struct SYS_MAP_MEMORY *p_map_mem; - struct mods_client *client = fp->private_data; struct list_head *head = &client->mem_map_list; struct list_head *iter; struct list_head *tmp; @@ -496,22 +512,22 @@ static void mods_unregister_all_mappings(struct file *fp) list_for_each_safe(iter, tmp, head) { p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list); - mods_unregister_mapping(fp, p_map_mem->virtual_addr); + mods_unregister_mapping(client, p_map_mem->virtual_addr); } LOG_EXT(); } -static pgprot_t mods_get_prot(u32 mem_type, pgprot_t prot) +static pgprot_t mods_get_prot(u8 mem_type, pgprot_t prot) { switch (mem_type) { - case MODS_MEMORY_CACHED: + case MODS_ALLOC_CACHED: return prot; - case MODS_MEMORY_UNCACHED: + case MODS_ALLOC_UNCACHED: return MODS_PGPROT_UC(prot); - case MODS_MEMORY_WRITECOMBINE: + case MODS_ALLOC_WRITECOMBINE: return MODS_PGPROT_WC(prot); default: @@ -521,11 +537,11 @@ static pgprot_t mods_get_prot(u32 mem_type, pgprot_t prot) } } -static pgprot_t mods_get_prot_for_range(struct file *fp, u64 dma_addr, - u64 size, pgprot_t prot) +static pgprot_t mods_get_prot_for_range(struct mods_client *client, + u64 dma_addr, + u64 size, + pgprot_t prot) { - struct mods_client *client = fp->private_data; - if ((dma_addr == client->mem_type.dma_addr) && (size == client->mem_type.size)) { @@ -534,16 +550,16 @@ static pgprot_t mods_get_prot_for_range(struct file *fp, u64 dma_addr, return prot; } -const char *mods_get_prot_str(u32 mem_type) +const char *mods_get_prot_str(u8 mem_type) { switch (mem_type) { - case MODS_MEMORY_CACHED: + case MODS_ALLOC_CACHED: return "WB"; - case MODS_MEMORY_UNCACHED: + case MODS_ALLOC_UNCACHED: return "UC"; - case MODS_MEMORY_WRITECOMBINE: + case MODS_ALLOC_WRITECOMBINE: return "WC"; default: @@ -551,12 +567,10 @@ const char *mods_get_prot_str(u32 mem_type) } } -static const char *mods_get_prot_str_for_range(struct file *fp, - u64 dma_addr, - u64 size) +static const char *mods_get_prot_str_for_range(struct mods_client *client, + u64 dma_addr, + u64 size) { - struct mods_client *client = fp->private_data; - if ((dma_addr == client->mem_type.dma_addr) && (size == client->mem_type.size)) { @@ -567,7 +581,7 @@ static const char *mods_get_prot_str_for_range(struct file *fp, /******************** * PCI ERROR FUNCTIONS * ********************/ -#if defined(MODS_CAN_REGISTER_PCI_DEV) +#if defined(CONFIG_PCI) static pci_ers_result_t mods_pci_error_detected(struct pci_dev *dev, enum pci_channel_state state) { @@ -609,7 +623,7 @@ static void mods_pci_resume(struct pci_dev *dev) ********************/ static void mods_krnl_vma_open(struct vm_area_struct *vma) { - struct mods_vm_private_data *vma_private_data; + struct mods_vm_private_data *priv; LOG_ENT(); mods_debug_printk(DEBUG_MEM_DETAILED, @@ -617,42 +631,41 @@ static void mods_krnl_vma_open(struct vm_area_struct *vma) vma->vm_start, (u64)(MODS_VMA_PGOFF(vma) << PAGE_SHIFT)); - if (MODS_VMA_PRIVATE(vma)) { - vma_private_data = MODS_VMA_PRIVATE(vma); - atomic_inc(&vma_private_data->usage_count); - } + priv = vma->vm_private_data; + if (priv) + atomic_inc(&priv->usage_count); + LOG_EXT(); } static void mods_krnl_vma_close(struct vm_area_struct *vma) { + struct mods_vm_private_data *priv; + LOG_ENT(); - if (MODS_VMA_PRIVATE(vma)) { - struct mods_vm_private_data *vma_private_data - = MODS_VMA_PRIVATE(vma); - if (atomic_dec_and_test(&vma_private_data->usage_count)) { - struct mods_client *client = - vma_private_data->fp->private_data; + priv = vma->vm_private_data; + if (priv && atomic_dec_and_test(&priv->usage_count)) { + struct mods_client *client = priv->client; - if (unlikely(mutex_lock_interruptible( - &client->mtx))) { - LOG_EXT(); - return; - } - - /* we need to unregister the mapping */ - mods_unregister_mapping(vma_private_data->fp, - vma->vm_start); - mods_debug_printk(DEBUG_MEM_DETAILED, - "closed vma, virt 0x%lx\n", - vma->vm_start); - MODS_VMA_PRIVATE(vma) = NULL; - kfree(vma_private_data); - - mutex_unlock(&client->mtx); + if (unlikely(mutex_lock_interruptible( + &client->mtx))) { + LOG_EXT(); + return; } + + /* we need to unregister the mapping */ + mods_unregister_mapping(client, vma->vm_start); + mods_debug_printk(DEBUG_MEM_DETAILED, + "closed vma, virt 0x%lx\n", + vma->vm_start); + vma->vm_private_data = NULL; + kfree(priv); + atomic_dec(&client->num_allocs); + + mutex_unlock(&client->mtx); } + LOG_EXT(); } @@ -684,78 +697,102 @@ static int mods_krnl_open(struct inode *ip, struct file *fp) static int mods_krnl_close(struct inode *ip, struct file *fp) { struct mods_client *client = fp->private_data; - u8 client_id = client->client_id; - int err = OK; + int final_err = OK; + int err = OK; LOG_ENT(); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (!validate_client(client)) { LOG_EXT(); return -EINVAL; } mods_free_client_interrupts(client); - mods_resume_console(fp); + mods_resume_console(client); - mods_unregister_all_mappings(fp); - err = mods_unregister_all_alloc(fp); + mods_unregister_all_mappings(client); + err = mods_unregister_all_alloc(client); if (err) mods_error_printk("failed to free all memory\n"); + final_err = err; #if defined(CONFIG_PPC64) - err = mods_unregister_all_ppc_tce_bypass(fp); + err = mods_unregister_all_ppc_tce_bypass(client); if (err) mods_error_printk("failed to restore dma bypass\n"); + if (!final_err) + final_err = err; - err = mods_unregister_all_nvlink_sysmem_trained(fp); + err = mods_unregister_all_nvlink_sysmem_trained(client); if (err) mods_error_printk("failed to free nvlink trained\n"); + if (!final_err) + final_err = err; #endif mods_disable_all_devices(client); - mods_free_client(client_id); + if (atomic_read(&client->num_allocs) || + atomic_read(&client->num_pages)) { + + mods_error_printk("not all allocations have been freed, allocs=%u, pages=%u\n", + (unsigned int)atomic_read(&client->num_allocs), + (unsigned int)atomic_read(&client->num_pages)); + if (!final_err) + final_err = -ENOMEM; + } + + mods_free_client(client->client_id); mods_info_printk("driver closed\n"); LOG_EXT(); - return err; + return final_err; } static unsigned int mods_krnl_poll(struct file *fp, poll_table *wait) { unsigned int mask = 0; struct mods_client *client = fp->private_data; - u8 client_id = get_client_id(fp); - int access_tok_ret = mods_check_access_token(fp); + int err; - if (access_tok_ret < 0) - return access_tok_ret; + if (!validate_client(client)) + return -EINVAL; + + err = mods_check_access_token(client); + if (err < 0) + return err; if (!(fp->f_flags & O_NONBLOCK)) { mods_debug_printk(DEBUG_ISR_DETAILED, "poll wait\n"); poll_wait(fp, &client->interrupt_event, wait); } /* if any interrupts pending then check intr, POLLIN on irq */ - mask |= mods_irq_event_check(client_id); + mask |= mods_irq_event_check(client->client_id); mods_debug_printk(DEBUG_ISR_DETAILED, "poll mask 0x%x\n", mask); return mask; } -static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma); +static int mods_krnl_map_inner(struct mods_client *client, + struct vm_area_struct *vma); static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma) { struct mods_vm_private_data *vma_private_data; - int access_tok_ret; + struct mods_client *client = fp->private_data; + int err; LOG_ENT(); - access_tok_ret = mods_check_access_token(fp); - if (access_tok_ret < 0) { + if (!validate_client(client)) { LOG_EXT(); - return access_tok_ret; + return -EINVAL; + } + + err = mods_check_access_token(client); + if (err < 0) { + LOG_EXT(); + return err; } vma->vm_ops = &mods_krnl_vm_ops; @@ -766,23 +803,23 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma) LOG_EXT(); return -ENOMEM; } + atomic_inc(&client->num_allocs); /* set private data for vm_area_struct */ atomic_set(&vma_private_data->usage_count, 0); - vma_private_data->fp = fp; - MODS_VMA_PRIVATE(vma) = vma_private_data; + vma_private_data->client = client; + vma->vm_private_data = vma_private_data; /* call for the first time open function */ mods_krnl_vma_open(vma); { int err = OK; - struct mods_client *client = fp->private_data; if (unlikely(mutex_lock_interruptible(&client->mtx))) err = -EINTR; else { - err = mods_krnl_map_inner(fp, vma); + err = mods_krnl_map_inner(client, vma); mutex_unlock(&client->mtx); } LOG_EXT(); @@ -790,10 +827,11 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma) } } -static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) +static int mods_krnl_map_inner(struct mods_client *client, + struct vm_area_struct *vma) { u64 req_pa = MODS_VMA_OFFSET(vma); - struct MODS_MEM_INFO *p_mem_info = mods_find_alloc(fp, req_pa); + struct MODS_MEM_INFO *p_mem_info = mods_find_alloc(client, req_pa); u32 req_pages = MODS_VMA_SIZE(vma) >> PAGE_SHIFT; if ((req_pa & ~PAGE_MASK) != 0 || @@ -885,7 +923,7 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) /* MODS_VMA_OFFSET(vma) can change so it can't be used * to register the mapping */ - mods_register_mapping(fp, + mods_register_mapping(client, p_mem_info, chunks[first].dma_addr, vma->vm_start, @@ -899,7 +937,7 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) req_pa, (unsigned long)vma->vm_start, (unsigned long)MODS_VMA_SIZE(vma), - mods_get_prot_str_for_range(fp, req_pa, + mods_get_prot_str_for_range(client, req_pa, MODS_VMA_SIZE(vma))); if (io_remap_pfn_range( @@ -908,7 +946,7 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) req_pa>>PAGE_SHIFT, MODS_VMA_SIZE(vma), mods_get_prot_for_range( - fp, + client, req_pa, MODS_VMA_SIZE(vma), vma->vm_page_prot))) { @@ -916,7 +954,7 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) return -EAGAIN; } - mods_register_mapping(fp, + mods_register_mapping(client, NULL, req_pa, vma->vm_start, @@ -944,28 +982,21 @@ static int mods_get_screen_info(struct MODS_SCREEN_INFO *p) * ESCAPE CALL FUNCTIONS * *************************/ -static int esc_mods_get_api_version(struct file *pfile, +static int esc_mods_get_api_version(struct mods_client *client, struct MODS_GET_VERSION *p) { p->version = MODS_DRIVER_VERSION; return OK; } -static int esc_mods_get_kernel_version(struct file *pfile, +static int esc_mods_get_kernel_version(struct mods_client *client, struct MODS_GET_VERSION *p) { p->version = MODS_KERNEL_VERSION; return OK; } -static int esc_mods_set_driver_para(struct file *pfile, - struct MODS_SET_PARA *p) -{ - int rc = OK; - return rc; -} - -static int esc_mods_get_screen_info(struct file *pfile, +static int esc_mods_get_screen_info(struct mods_client *client, struct MODS_SCREEN_INFO *p) { #if defined(CONFIG_ARM) || defined(CONFIG_ARM64) || defined(CONFIG_PPC64) @@ -982,7 +1013,7 @@ static int esc_mods_get_screen_info(struct file *pfile, #endif } -static int esc_mods_get_screen_info_2(struct file *pfile, +static int esc_mods_get_screen_info_2(struct mods_client *client, struct MODS_SCREEN_INFO_2 *p) { #if defined(CONFIG_ARM) || defined(CONFIG_ARM64) || defined(CONFIG_PPC64) @@ -1000,7 +1031,7 @@ static int esc_mods_get_screen_info_2(struct file *pfile, #endif } -static int esc_mods_lock_console(struct file *pfile) +static int esc_mods_lock_console(struct mods_client *client) { #if defined(MODS_HAS_CONSOLE_LOCK) console_lock(); @@ -1010,7 +1041,7 @@ static int esc_mods_lock_console(struct file *pfile) #endif } -static int esc_mods_unlock_console(struct file *pfile) +static int esc_mods_unlock_console(struct mods_client *client) { #if defined(MODS_HAS_CONSOLE_LOCK) console_unlock(); @@ -1020,7 +1051,7 @@ static int esc_mods_unlock_console(struct file *pfile) #endif } -static int esc_mods_suspend_console(struct file *pfile) +static int esc_mods_suspend_console(struct mods_client *client) { int err = -EINVAL; @@ -1029,7 +1060,6 @@ static int esc_mods_suspend_console(struct file *pfile) #if defined(CONFIG_FB) && defined(MODS_HAS_CONSOLE_LOCK) if (num_registered_fb) { /* tell the os to block fb accesses */ - struct mods_client *client = pfile->private_data; int i = 0; for (i = 0; i < num_registered_fb; i++) { @@ -1061,12 +1091,12 @@ static int esc_mods_suspend_console(struct file *pfile) return err; } -static int esc_mods_resume_console(struct file *pfile) +static int esc_mods_resume_console(struct mods_client *client) { - return mods_resume_console(pfile); + return mods_resume_console(client); } -static int mods_resume_console(struct file *pfile) +static int mods_resume_console(struct mods_client *client) { int err = -EINVAL; @@ -1074,7 +1104,6 @@ static int mods_resume_console(struct file *pfile) #if defined(CONFIG_FB) && defined(MODS_HAS_CONSOLE_LOCK) if (num_registered_fb) { - struct mods_client *client = pfile->private_data; int i = 0; for (i = 0; i < num_registered_fb; i++) { @@ -1106,7 +1135,7 @@ static int mods_resume_console(struct file *pfile) return err; } -static int esc_mods_acquire_access_token(struct file *pfile, +static int esc_mods_acquire_access_token(struct mods_client *client, struct MODS_ACCESS_TOKEN *ptoken) { int err = -EINVAL; @@ -1124,18 +1153,15 @@ static int esc_mods_acquire_access_token(struct file *pfile, err = mods_set_access_token(ptoken->token); if (err) mods_error_printk("unable to set access token!\n"); - else { - struct mods_client *client = pfile->private_data; - + else client->access_token = ptoken->token; - } LOG_EXT(); return err; } -static int esc_mods_release_access_token(struct file *pfile, +static int esc_mods_release_access_token(struct mods_client *client, struct MODS_ACCESS_TOKEN *ptoken) { int err = -EINVAL; @@ -1152,18 +1178,15 @@ static int esc_mods_release_access_token(struct file *pfile, err = mods_set_access_token(MODS_ACCESS_TOKEN_NONE); if (err) mods_error_printk("unable to clear access token!\n"); - else { - struct mods_client *client = pfile->private_data; - + else client->access_token = MODS_ACCESS_TOKEN_NONE; - } LOG_EXT(); return err; } -static int esc_mods_verify_access_token(struct file *pfile, +static int esc_mods_verify_access_token(struct mods_client *client, struct MODS_ACCESS_TOKEN *ptoken) { int err = -EINVAL; @@ -1171,8 +1194,6 @@ static int esc_mods_verify_access_token(struct file *pfile, LOG_ENT(); if (ptoken->token == mods_get_access_token()) { - struct mods_client *client = pfile->private_data; - client->access_token = ptoken->token; err = OK; } else @@ -1189,11 +1210,7 @@ struct mods_sysfs_work { int err; }; -#ifdef MODS_OLD_INIT_WORK -static void sysfs_write_task(void *w) -#else static void sysfs_write_task(struct work_struct *w) -#endif { struct mods_sysfs_work *task = container_of(w, struct mods_sysfs_work, @@ -1226,7 +1243,7 @@ static void sysfs_write_task(struct work_struct *w) LOG_EXT(); } -static int esc_mods_write_sysfs_node(struct file *pfile, +static int esc_mods_write_sysfs_node(struct mods_client *client, struct MODS_SYSFS_NODE *pdata) { int err = -EINVAL; @@ -1247,11 +1264,7 @@ static int esc_mods_write_sysfs_node(struct file *pfile, return err; } -#ifdef MODS_OLD_INIT_WORK - INIT_WORK(&task.work, sysfs_write_task, &task); -#else INIT_WORK(&task.work, sysfs_write_task); -#endif queue_work(wq, &task.work); flush_workqueue(wq); destroy_workqueue(wq); @@ -1264,8 +1277,8 @@ static int esc_mods_write_sysfs_node(struct file *pfile, return err; } -#ifdef MODS_HAS_MSR -static int esc_mods_read_msr(struct file *pfile, struct MODS_MSR *p) +#ifdef CONFIG_X86 +static int esc_mods_read_msr(struct mods_client *client, struct MODS_MSR *p) { int err = -EINVAL; @@ -1279,7 +1292,7 @@ static int esc_mods_read_msr(struct file *pfile, struct MODS_MSR *p) return err; } -static int esc_mods_write_msr(struct file *pfile, struct MODS_MSR *p) +static int esc_mods_write_msr(struct mods_client *client, struct MODS_MSR *p) { int err = -EINVAL; @@ -1302,17 +1315,23 @@ static long mods_krnl_ioctl(struct file *fp, unsigned int cmd, unsigned long i_arg) { - int err = 0; - void *arg_copy = 0; - void *arg = (void *) i_arg; - int arg_size; - char buf[64]; + int err = 0; + void *arg_copy = NULL; + void *arg = (void *)i_arg; + struct mods_client *client = fp->private_data; + int arg_size; + char buf[64]; LOG_ENT(); + if (!validate_client(client)) { + LOG_EXT(); + return -EINVAL; + } + if ((cmd != MODS_ESC_VERIFY_ACCESS_TOKEN) && (cmd != MODS_ESC_GET_API_VERSION)) { - err = mods_check_access_token(fp); + err = mods_check_access_token(client); if (err) { LOG_EXT(); return err; @@ -1327,12 +1346,16 @@ static long mods_krnl_ioctl(struct file *fp, LOG_EXT(); return -ENOMEM; } + atomic_inc(&client->num_allocs); } else if (arg_size > 0) arg_copy = buf; if ((arg_size > 0) && copy_from_user(arg_copy, arg, arg_size)) { mods_error_printk("failed to copy ioctl data\n"); - kfree(arg_copy); + if (arg_size > (int)sizeof(buf)) { + kfree(arg_copy); + atomic_dec(&client->num_allocs); + } LOG_EXT(); return -EFAULT; } @@ -1347,7 +1370,7 @@ static long mods_krnl_ioctl(struct file *fp, "invalid parameter passed to ioctl " #code \ "\n");\ } else {\ - err = function(fp, (struct argtype *)arg_copy);\ + err = function(client, (struct argtype *)arg_copy);\ if ((err == OK) && \ copy_to_user(arg, arg_copy, arg_size)) {\ err = -EFAULT;\ @@ -1369,7 +1392,7 @@ static long mods_krnl_ioctl(struct file *fp, "invalid parameter passed to ioctl " #code \ "\n");\ } else {\ - err = function(fp, (struct argtype *)arg_copy);\ + err = function(client, (struct argtype *)arg_copy);\ } \ } while (0);\ }) @@ -1384,7 +1407,7 @@ static long mods_krnl_ioctl(struct file *fp, "invalid parameter passed to ioctl " #code \ "\n");\ } else {\ - err = function(fp);\ + err = function(client);\ } \ } while (0);\ }) @@ -1471,6 +1494,12 @@ static long mods_krnl_ioctl(struct file *fp, MODS_PCI_HOT_RESET); break; + case MODS_ESC_PCI_BUS_REMOVE_DEV: + MODS_IOCTL_NORETVAL(MODS_ESC_PCI_BUS_REMOVE_DEV, + esc_mods_pci_bus_remove_dev, + MODS_PCI_BUS_REMOVE_DEV); + break; + case MODS_ESC_PIO_READ: MODS_IOCTL(MODS_ESC_PIO_READ, esc_mods_pio_read, MODS_PIO_READ); @@ -1529,6 +1558,12 @@ static long mods_krnl_ioctl(struct file *fp, MODS_DEVICE_ALLOC_PAGES_2); break; + case MODS_ESC_ALLOC_PAGES_2: + MODS_IOCTL(MODS_ESC_ALLOC_PAGES_2, + esc_mods_alloc_pages_2, + MODS_ALLOC_PAGES_2); + break; + case MODS_ESC_FREE_PAGES: MODS_IOCTL(MODS_ESC_FREE_PAGES, esc_mods_free_pages, MODS_FREE_PAGES); @@ -1738,11 +1773,6 @@ static long mods_krnl_ioctl(struct file *fp, esc_mods_get_kernel_version, MODS_GET_VERSION); break; - case MODS_ESC_SET_DRIVER_PARA: - MODS_IOCTL_NORETVAL(MODS_ESC_SET_DRIVER_PARA, - esc_mods_set_driver_para, MODS_SET_PARA); - break; - #if defined(MODS_HAS_CLOCK) case MODS_ESC_GET_CLOCK_HANDLE: MODS_IOCTL(MODS_ESC_GET_CLOCK_HANDLE, @@ -2018,7 +2048,7 @@ static long mods_krnl_ioctl(struct file *fp, break; #endif -#ifdef MODS_HAS_MSR +#ifdef CONFIG_X86 case MODS_ESC_READ_MSR: MODS_IOCTL(MODS_ESC_READ_MSR, esc_mods_read_msr, MODS_MSR); @@ -2038,8 +2068,10 @@ static long mods_krnl_ioctl(struct file *fp, break; } - if (arg_size > (int)sizeof(buf)) + if (arg_size > (int)sizeof(buf)) { kfree(arg_copy); + atomic_dec(&client->num_allocs); + } LOG_EXT(); return err; diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c index 3f028f95..2f9fba21 100644 --- a/drivers/misc/mods/mods_mem.c +++ b/drivers/misc/mods/mods_mem.c @@ -2,7 +2,7 @@ /* * mods_mem.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -30,8 +30,6 @@ static int mods_post_alloc(struct MODS_PHYS_CHUNK *chunk, u64 phys_addr, struct MODS_MEM_INFO *p_mem_info); -static void mods_pre_free(struct MODS_PHYS_CHUNK *chunk, - struct MODS_MEM_INFO *p_mem_info); /**************************** * DMA MAP HELPER FUNCTIONS * @@ -121,7 +119,8 @@ static void mods_dma_unmap_page(struct pci_dev *dev, } /* Unmap and delete the specified DMA mapping */ -static int mods_dma_unmap_and_free(struct MODS_MEM_INFO *p_mem_info, +static int mods_dma_unmap_and_free(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info, struct MODS_DMA_MAP *p_del_map) { @@ -161,13 +160,15 @@ static int mods_dma_unmap_and_free(struct MODS_MEM_INFO *p_mem_info, } kfree(p_del_map); + atomic_dec(&client->num_allocs); return OK; } #endif /* Unmap and delete all DMA mappings on the specified allocation */ -int mods_dma_unmap_all(struct MODS_MEM_INFO *p_mem_info, +int mods_dma_unmap_all(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info, struct pci_dev *dev) { #ifdef CONFIG_PCI @@ -182,7 +183,9 @@ int mods_dma_unmap_all(struct MODS_MEM_INFO *p_mem_info, p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list); if (!dev || (p_dma_map->dev == dev)) { - err = mods_dma_unmap_and_free(p_mem_info, p_dma_map); + err = mods_dma_unmap_and_free(client, + p_mem_info, + p_dma_map); if (err || dev) break; } @@ -212,7 +215,7 @@ static int mods_dma_map_pages(struct MODS_MEM_INFO *p_mem_info, PAGE_SIZE << chunk->order, DMA_BIDIRECTIONAL); - if (MODS_PCI_DMA_MAPPING_ERROR(dev, dev_addr)) { + if (pci_dma_mapping_error(dev, dev_addr)) { mods_error_printk("failed to map page to device %04x:%02x:%02x.%x\n", pci_domain_nr(dev->bus), dev->bus->number, @@ -247,7 +250,8 @@ static int mods_dma_map_pages(struct MODS_MEM_INFO *p_mem_info, /* Create a DMA map on the specified allocation for the pci device. * Lazy-initialize the map list structure if one does not yet exist. */ -static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, +static int mods_create_dma_map(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info, struct pci_dev *dev) { struct MODS_DMA_MAP *p_dma_map; @@ -262,6 +266,7 @@ static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, mods_error_printk("failed to allocate device map data\n"); return -ENOMEM; } + atomic_inc(&client->num_allocs); p_dma_map->dev = pci_dev_get(dev); err = mods_dma_map_pages(p_mem_info, p_dma_map); @@ -269,6 +274,7 @@ static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, if (unlikely(err)) { pci_dev_put(dev); kfree(p_dma_map); + atomic_dec(&client->num_allocs); } else list_add(&p_dma_map->list, &p_mem_info->dma_map_list); @@ -284,7 +290,7 @@ static int mods_dma_map_default_page(struct MODS_PHYS_CHUNK *chunk, PAGE_SIZE << chunk->order, DMA_BIDIRECTIONAL); - if (MODS_PCI_DMA_MAPPING_ERROR(dev, dev_addr)) { + if (pci_dma_mapping_error(dev, dev_addr)) { mods_error_printk("failed to map page to device %04x:%02x:%02x.%x\n", pci_domain_nr(dev->bus), dev->bus->number, @@ -362,45 +368,179 @@ static struct MODS_DMA_MAP *find_dma_map(struct MODS_MEM_INFO *p_mem_info, } #if !defined(MODS_TEGRA) || defined(CONFIG_CPA) -static int mods_set_mem_type(u64 virt_addr, u64 pages, u32 type) +static int mods_set_mem_type(u64 virt_addr, u64 pages, u8 type) { - if (type == MODS_MEMORY_UNCACHED) + if (type == MODS_ALLOC_UNCACHED) return MODS_SET_MEMORY_UC(virt_addr, pages); - else if (type == MODS_MEMORY_WRITECOMBINE) + else if (type == MODS_ALLOC_WRITECOMBINE) return MODS_SET_MEMORY_WC(virt_addr, pages); return 0; } #endif -static int mods_restore_mem_type(u64 virt_addr, - u64 pages, - u32 type_override) +/* In order to map pages as UC or WC to the CPU, we need to change their + * attributes by calling set_memory_uc()/set_memory_wc(), respectively. + * On some CPUs this operation is extremely slow. In order to incur + * this penalty only once, we save pages mapped as UC or WC so that + * we can reuse them later. + */ +static int save_non_wb_chunks(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info) { - if ((type_override == MODS_MEMORY_UNCACHED) || - (type_override == MODS_MEMORY_WRITECOMBINE)) { - return MODS_SET_MEMORY_WB(virt_addr, pages); + u32 ichunk; + int err = 0; + + if (p_mem_info->cache_type == MODS_ALLOC_CACHED) + return 0; + + if (unlikely(mutex_lock_interruptible(&client->mtx))) + return -EINTR; + + /* Steal the chunks from MODS_MEM_INFO and put them on free list. */ + + for (ichunk = 0; ichunk < p_mem_info->num_chunks; ichunk++) { + + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[ichunk]; + struct MODS_FREE_PHYS_CHUNK *free_chunk; + + if (!chunk->wc) + continue; + + free_chunk = kzalloc(sizeof(struct MODS_FREE_PHYS_CHUNK), + GFP_KERNEL | __GFP_NORETRY); + + if (!free_chunk) { + err = -ENOMEM; + break; + } + atomic_inc(&client->num_allocs); + + free_chunk->numa_node = p_mem_info->numa_node; + free_chunk->order = chunk->order; + free_chunk->cache_type = p_mem_info->cache_type; + free_chunk->dma32 = p_mem_info->dma32; + free_chunk->p_page = chunk->p_page; + + chunk->p_page = NULL; + + mods_debug_printk(DEBUG_MEM_DETAILED, + "save 0x%llx 2^%d pages %s\n", + (unsigned long long)(size_t)free_chunk->p_page, + chunk->order, + p_mem_info->cache_type == MODS_ALLOC_WRITECOMBINE + ? "WC" : "UC"); + +#ifdef CONFIG_PCI + if (chunk->mapped) { + mods_dma_unmap_page(p_mem_info->dev, + chunk->dev_addr, + chunk->order); + chunk->mapped = 0; + } +#endif + + list_add(&free_chunk->list, &client->free_mem_list); } - return 0; + + mutex_unlock(&client->mtx); + + return err; } -static void mods_restore_cache(struct MODS_MEM_INFO *p_mem_info) +static int mods_restore_cache_one_chunk(struct page *p_page, u8 order) +{ + int final_err = 0; + u32 num_pages = 1U << order; + u32 i; + + for (i = 0; i < num_pages; i++) { + u64 ptr = (u64)(size_t)kmap(p_page + i); + + if (ptr) { + int err = MODS_SET_MEMORY_WB(ptr, 1); + + if (!final_err) + final_err = err; + } + + kunmap((void *)(size_t)ptr); + } + + return final_err; +} + +static int release_free_chunks(struct mods_client *client) +{ + struct list_head *head; + struct list_head *iter; + struct list_head *next; + int final_err = 0; + + if (unlikely(mutex_lock_interruptible(&client->mtx))) + return -EINTR; + + head = &client->free_mem_list; + + list_for_each_safe(iter, next, head) { + + struct MODS_FREE_PHYS_CHUNK *free_chunk; + int err; + + free_chunk = list_entry(iter, + struct MODS_FREE_PHYS_CHUNK, + list); + + list_del(iter); + + err = mods_restore_cache_one_chunk(free_chunk->p_page, + free_chunk->order); + + if (!final_err) + final_err = err; + + __free_pages(free_chunk->p_page, free_chunk->order); + atomic_sub(1u << free_chunk->order, &client->num_pages); + + kfree(free_chunk); + atomic_dec(&client->num_allocs); + } + + mutex_unlock(&client->mtx); + + return final_err; +} + +static int mods_restore_cache(struct MODS_MEM_INFO *p_mem_info) { unsigned int i; + int final_err = 0; + + if (p_mem_info->cache_type == MODS_ALLOC_CACHED) + return 0; for (i = 0; i < p_mem_info->num_chunks; i++) { + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[i]; + int err; - if (!chunk->p_page) - break; + if (!chunk->p_page || !chunk->wc) + continue; - mods_pre_free(chunk, p_mem_info); + err = mods_restore_cache_one_chunk(chunk->p_page, chunk->order); + if (!final_err) + final_err = err; } + + return final_err; } -static void mods_free_pages(struct MODS_MEM_INFO *p_mem_info) +static void mods_free_pages(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info) { unsigned int i; + mods_restore_cache(p_mem_info); + /* release in reverse order */ for (i = p_mem_info->num_chunks; i > 0; ) { struct MODS_PHYS_CHUNK *chunk; @@ -420,23 +560,24 @@ static void mods_free_pages(struct MODS_MEM_INFO *p_mem_info) #endif __free_pages(chunk->p_page, chunk->order); + atomic_sub(1u << chunk->order, &client->num_pages); chunk->p_page = 0; } } -static gfp_t mods_alloc_flags(struct MODS_MEM_INFO *p_mem_info) +static gfp_t mods_alloc_flags(struct MODS_MEM_INFO *p_mem_info, u32 order) { gfp_t flags = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; -#ifdef MODS_HAS_DEV_TO_NUMA_NODE - flags |= __GFP_THISNODE; -#endif + if (p_mem_info->force_numa) + flags |= __GFP_THISNODE; - if (p_mem_info->alloc_type != MODS_ALLOC_TYPE_NON_CONTIG) + if (order) flags |= __GFP_COMP; - if (p_mem_info->addr_bits == 32) -#ifdef MODS_HAS_DMA32 + + if (p_mem_info->dma32) +#ifdef CONFIG_ZONE_DMA32 flags |= __GFP_DMA32; #else flags |= __GFP_DMA; @@ -447,11 +588,79 @@ static gfp_t mods_alloc_flags(struct MODS_MEM_INFO *p_mem_info) return flags; } -static int mods_alloc_contig_sys_pages(struct MODS_MEM_INFO *p_mem_info) +static struct page *mods_alloc_pages(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info, + u32 order, + int *need_cup) { - u64 phys_addr; - u64 end_addr = 0; - u32 order = 0; + struct page *p_page = NULL; + u8 cache_type = p_mem_info->cache_type; + u8 dma32 = p_mem_info->dma32; + int numa_node = p_mem_info->numa_node; + + if (likely(!mutex_lock_interruptible(&client->mtx))) { + + struct list_head *iter; + struct list_head *head = &client->free_mem_list; + struct MODS_FREE_PHYS_CHUNK *free_chunk = NULL; + + list_for_each(iter, head) { + free_chunk = list_entry(iter, + struct MODS_FREE_PHYS_CHUNK, + list); + + if (free_chunk->cache_type == cache_type && + free_chunk->dma32 == dma32 && + free_chunk->numa_node == numa_node && + free_chunk->order == order) { + + list_del(iter); + break; + } + + free_chunk = NULL; + } + + mutex_unlock(&client->mtx); + + if (free_chunk) { + p_page = free_chunk->p_page; + kfree(free_chunk); + atomic_dec(&client->num_allocs); + + mods_debug_printk(DEBUG_MEM_DETAILED, + "reuse 0x%llx 2^%d pages %s\n", + (unsigned long long)(size_t)p_page, + order, + cache_type == MODS_ALLOC_WRITECOMBINE + ? "WC" : "UC"); + + *need_cup = 0; + return p_page; + } + } + + p_page = alloc_pages_node(p_mem_info->numa_node, + mods_alloc_flags(p_mem_info, order), + order); + + *need_cup = 1; + + if (p_page) + atomic_add(1u << order, &client->num_pages); + + return p_page; +} + +static int mods_alloc_contig_sys_pages(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info) +{ + u64 phys_addr; + u64 dma_addr; + u64 end_addr = 0; + u32 order = 0; + int is_wb = 1; + struct page *p_page; LOG_ENT(); @@ -459,48 +668,60 @@ static int mods_alloc_contig_sys_pages(struct MODS_MEM_INFO *p_mem_info) order++; p_mem_info->pages[0].order = order; - p_mem_info->pages[0].p_page = alloc_pages_node( - p_mem_info->numa_node, - mods_alloc_flags(p_mem_info), - order); + p_page = mods_alloc_pages(client, p_mem_info, order, &is_wb); - if (!p_mem_info->pages[0].p_page && - p_mem_info->logical_addr == 0) { + if (!p_page && p_mem_info->logical_addr == 0) { LOG_EXT(); return -ENOMEM; } - phys_addr = page_to_phys(p_mem_info->pages[0].p_page); + p_mem_info->pages[0].p_page = p_page; + + if (!is_wb) + p_mem_info->pages[0].wc = 1; + + phys_addr = page_to_phys(p_page); if (phys_addr == 0) { mods_error_printk("failed to determine physical address\n"); - mods_free_pages(p_mem_info); + mods_free_pages(client, p_mem_info); p_mem_info->logical_addr = 0; LOG_EXT(); return -ENOMEM; } - p_mem_info->pages[0].dma_addr = MODS_PHYS_TO_DMA(phys_addr); + dma_addr = MODS_PHYS_TO_DMA(phys_addr); + + if (dma_addr >= (1ULL << DMA_BITS)) { + mods_error_printk("dma_addr 0x%llx exceeds supported range\n", + dma_addr); + mods_free_pages(client, p_mem_info); + p_mem_info->logical_addr = 0; + LOG_EXT(); + return -ENOMEM; + } + + p_mem_info->pages[0].dma_addr = dma_addr; mods_debug_printk(DEBUG_MEM, - "alloc contig 0x%lx bytes, 2^%u pages, %s, node %d, addrbits %u, phys 0x%llx\n", + "alloc contig 0x%lx bytes, 2^%u pages, %s, node %d,%s phys 0x%llx\n", (unsigned long)p_mem_info->length, p_mem_info->pages[0].order, mods_get_prot_str(p_mem_info->cache_type), p_mem_info->numa_node, - (unsigned int)p_mem_info->addr_bits, - (unsigned long long)p_mem_info->pages[0].dma_addr); + p_mem_info->dma32 ? " dma32," : "", + (unsigned long long)dma_addr); - end_addr = p_mem_info->pages[0].dma_addr + p_mem_info->length; - if ((p_mem_info->addr_bits == 32) && + end_addr = dma_addr + p_mem_info->length; + if ((p_mem_info->dma32) && (end_addr > 0x100000000ULL)) { mods_error_printk("allocation exceeds 32-bit addressing\n"); - mods_free_pages(p_mem_info); + mods_free_pages(client, p_mem_info); p_mem_info->logical_addr = 0; LOG_EXT(); return -ENOMEM; } if (mods_post_alloc(p_mem_info->pages, phys_addr, p_mem_info)) { - mods_free_pages(p_mem_info); + mods_free_pages(client, p_mem_info); p_mem_info->logical_addr = 0; LOG_EXT(); return -EINVAL; @@ -518,7 +739,8 @@ static u32 mods_get_max_order_needed(u32 num_pages) return order; } -static int mods_alloc_noncontig_sys_pages(struct MODS_MEM_INFO *p_mem_info) +static int mods_alloc_noncontig_sys_pages(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info) { u32 pages_left = p_mem_info->num_pages; u32 num_chunks = 0; @@ -531,14 +753,16 @@ static int mods_alloc_noncontig_sys_pages(struct MODS_MEM_INFO *p_mem_info) /* alloc pages */ while (pages_left > 0) { u64 phys_addr = 0; + u64 dma_addr = 0; u32 order = mods_get_max_order_needed(pages_left); + int is_wb = 1; struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[num_chunks]; for (;;) { - chunk->p_page = alloc_pages_node( - p_mem_info->numa_node, - mods_alloc_flags(p_mem_info), - order); + chunk->p_page = mods_alloc_pages(client, + p_mem_info, + order, + &is_wb); if (chunk->p_page) break; if (order == 0) @@ -551,6 +775,9 @@ static int mods_alloc_noncontig_sys_pages(struct MODS_MEM_INFO *p_mem_info) goto failed; } + if (!is_wb) + chunk->wc = 1; + pages_left -= 1U << order; chunk->order = order; @@ -559,15 +786,23 @@ static int mods_alloc_noncontig_sys_pages(struct MODS_MEM_INFO *p_mem_info) mods_error_printk("phys addr lookup failed\n"); goto failed; } - chunk->dma_addr = MODS_PHYS_TO_DMA(phys_addr); + dma_addr = MODS_PHYS_TO_DMA(phys_addr); + + if (dma_addr >= (1ULL << DMA_BITS)) { + mods_error_printk("dma_addr 0x%llx exceeds supported range\n", + dma_addr); + goto failed; + } + + chunk->dma_addr = dma_addr; mods_debug_printk(DEBUG_MEM, - "alloc 0x%lx bytes [%u], 2^%u pages, %s, node %d, addrbits %u, phys 0x%llx\n", + "alloc 0x%lx bytes [%u], 2^%u pages, %s, node %d,%s phys 0x%llx\n", (unsigned long)p_mem_info->length, (unsigned int)num_chunks, chunk->order, mods_get_prot_str(p_mem_info->cache_type), p_mem_info->numa_node, - (unsigned int)p_mem_info->addr_bits, + p_mem_info->dma32 ? " dma32," : "", (unsigned long long)chunk->dma_addr); ++num_chunks; @@ -579,28 +814,25 @@ static int mods_alloc_noncontig_sys_pages(struct MODS_MEM_INFO *p_mem_info) return 0; failed: - mods_restore_cache(p_mem_info); - mods_free_pages(p_mem_info); + mods_free_pages(client, p_mem_info); return -ENOMEM; } -static int mods_register_alloc(struct file *fp, +static int mods_register_alloc(struct mods_client *client, struct MODS_MEM_INFO *p_mem_info) { - struct mods_client *client = fp->private_data; - if (unlikely(mutex_lock_interruptible(&client->mtx))) return -EINTR; + list_add(&p_mem_info->list, &client->mem_alloc_list); mutex_unlock(&client->mtx); return OK; } -static int mods_unregister_and_free(struct file *fp, +static int mods_unregister_and_free(struct mods_client *client, struct MODS_MEM_INFO *p_del_mem) { struct MODS_MEM_INFO *p_mem_info; - struct mods_client *client = fp->private_data; struct list_head *head; struct list_head *iter; @@ -619,12 +851,13 @@ static int mods_unregister_and_free(struct file *fp, mutex_unlock(&client->mtx); - mods_dma_unmap_all(p_mem_info, NULL); - mods_restore_cache(p_mem_info); - mods_free_pages(p_mem_info); + mods_dma_unmap_all(client, p_mem_info, NULL); + save_non_wb_chunks(client, p_mem_info); + mods_free_pages(client, p_mem_info); pci_dev_put(p_mem_info->dev); kfree(p_mem_info); + atomic_dec(&client->num_allocs); return OK; } @@ -637,27 +870,32 @@ static int mods_unregister_and_free(struct file *fp, return -EINVAL; } -int mods_unregister_all_alloc(struct file *fp) +int mods_unregister_all_alloc(struct mods_client *client) { - int err = OK; - struct mods_client *client = fp->private_data; - struct list_head *head = &client->mem_alloc_list; - struct list_head *iter; - struct list_head *tmp; + int final_err = OK; + int err; + struct list_head *head = &client->mem_alloc_list; + struct list_head *iter; + struct list_head *tmp; list_for_each_safe(iter, tmp, head) { + struct MODS_MEM_INFO *p_mem_info; p_mem_info = list_entry(iter, struct MODS_MEM_INFO, list); - err = mods_unregister_and_free(fp, p_mem_info); - if (err) - break; + err = mods_unregister_and_free(client, p_mem_info); + if (!final_err) + final_err = err; } - return err; + err = release_free_chunks(client); + if (!final_err) + final_err = err; + + return final_err; } -static int get_addr_range(struct file *fp, +static int get_addr_range(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS_3 *p, struct mods_pci_dev_2 *pcidev) { @@ -789,13 +1027,12 @@ int mods_get_alloc_offset(struct MODS_MEM_INFO *p_mem_info, return -EINVAL; } -struct MODS_MEM_INFO *mods_find_alloc(struct file *fp, u64 phys_addr) +struct MODS_MEM_INFO *mods_find_alloc(struct mods_client *client, u64 phys_addr) { - struct mods_client *client = fp->private_data; struct list_head *plist_head = &client->mem_alloc_list; struct list_head *plist_iter; struct MODS_MEM_INFO *p_mem_info; - u64 offset; + u64 offset; list_for_each(plist_iter, plist_head) { p_mem_info = list_entry(plist_iter, @@ -837,7 +1074,8 @@ static u32 mods_estimate_num_chunks(u32 num_pages) * chunks than originally estimated. This function reallocates the * MODS_MEM_INFO struct so that it uses only as much memory as it needs. */ -static struct MODS_MEM_INFO *optimize_chunks(struct MODS_MEM_INFO *p_mem_info) +static struct MODS_MEM_INFO *optimize_chunks(struct mods_client *client, + struct MODS_MEM_INFO *p_mem_info) { u32 i; u32 num_chunks; @@ -856,6 +1094,7 @@ static struct MODS_MEM_INFO *optimize_chunks(struct MODS_MEM_INFO *p_mem_info) p_new_mem_info = kzalloc(alloc_size, GFP_KERNEL | __GFP_NORETRY); + atomic_inc(&client->num_allocs); } if (p_new_mem_info) { @@ -863,6 +1102,7 @@ static struct MODS_MEM_INFO *optimize_chunks(struct MODS_MEM_INFO *p_mem_info) p_new_mem_info->num_chunks = num_chunks; INIT_LIST_HEAD(&p_new_mem_info->dma_map_list); kfree(p_mem_info); + atomic_dec(&client->num_allocs); p_mem_info = p_new_mem_info; } @@ -873,15 +1113,14 @@ static struct MODS_MEM_INFO *optimize_chunks(struct MODS_MEM_INFO *p_mem_info) * ESCAPE CALL FUNCTONS * ************************/ -int esc_mods_device_alloc_pages_2(struct file *fp, - struct MODS_DEVICE_ALLOC_PAGES_2 *p) +int esc_mods_alloc_pages_2(struct mods_client *client, + struct MODS_ALLOC_PAGES_2 *p) { struct MODS_MEM_INFO *p_mem_info = NULL; - u32 num_pages; - u32 alloc_size; - u32 num_chunks; - int err = OK; - struct pci_dev *dev = NULL; + u32 num_pages; + u32 alloc_size; + u32 num_chunks; + int err = OK; LOG_ENT(); @@ -892,38 +1131,48 @@ int esc_mods_device_alloc_pages_2(struct file *fp, } mods_debug_printk(DEBUG_MEM_DETAILED, - "alloc 0x%x bytes %s %s on %04x:%02x:%02x.%x\n", - p->num_bytes, - p->contiguous ? "contiguous" : "noncontiguous", - mods_get_prot_str(p->attrib), + "alloc 0x%llx bytes flags=0x%x (%s %s%s%s%s%s) node=%d on %04x:%02x:%02x.%x\n", + (unsigned long long)p->num_bytes, + p->flags, + mods_get_prot_str(p->flags & MODS_ALLOC_CACHE_MASK), + (p->flags & MODS_ALLOC_CONTIGUOUS) + ? "contiguous" : "noncontiguous", + (p->flags & MODS_ALLOC_DMA32) ? " dma32" : "", + (p->flags & MODS_ALLOC_USE_NUMA) ? " usenuma" : "", + (p->flags & MODS_ALLOC_FORCE_NUMA) + ? " forcenuma" : "", + (p->flags & MODS_ALLOC_MAP_DEV) ? " dmamap" : "", + p->numa_node, p->pci_device.domain, p->pci_device.bus, p->pci_device.device, p->pci_device.function); - switch (p->attrib) { - case MODS_MEMORY_CACHED: -#if !defined(CONFIG_PPC64) - case MODS_MEMORY_UNCACHED: - case MODS_MEMORY_WRITECOMBINE: -#endif - break; - - default: - mods_error_printk("invalid memory type: %u\n", - p->attrib); - err = -ENOMEM; - goto failed; - } - - num_pages = (u32)(((u64)p->num_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT); - if (p->contiguous) + num_pages = (u32)((p->num_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT); + if (p->flags & MODS_ALLOC_CONTIGUOUS) num_chunks = 1; else num_chunks = mods_estimate_num_chunks(num_pages); alloc_size = sizeof(*p_mem_info) + (num_chunks - 1) * sizeof(struct MODS_PHYS_CHUNK); + if (((u64)num_pages << PAGE_SHIFT) < p->num_bytes) { + mods_error_printk("invalid allocation size requested: 0x%llx\n", + (unsigned long long)p->num_bytes); + err = -EINVAL; + goto failed; + } + + if ((p->flags & MODS_ALLOC_USE_NUMA) && + (p->numa_node != MODS_ANY_NUMA_NODE) && + ((unsigned int)p->numa_node >= + (unsigned int)num_possible_nodes())) { + + mods_error_printk("invalid NUMA node: %d\n", p->numa_node); + err = -EINVAL; + goto failed; + } + p_mem_info = kzalloc(alloc_size, GFP_KERNEL | __GFP_NORETRY); if (unlikely(!p_mem_info)) { mods_error_printk("failed to allocate auxiliary 0x%x bytes\n", @@ -931,31 +1180,57 @@ int esc_mods_device_alloc_pages_2(struct file *fp, err = -ENOMEM; goto failed; } + atomic_inc(&client->num_allocs); p_mem_info->num_chunks = num_chunks; - p_mem_info->alloc_type = p->contiguous - ? MODS_ALLOC_TYPE_CONTIG : MODS_ALLOC_TYPE_NON_CONTIG; - p_mem_info->cache_type = p->attrib; - p_mem_info->length = p->num_bytes; + p_mem_info->length = p->num_bytes; p_mem_info->logical_addr = 0; - p_mem_info->addr_bits = p->address_bits; - p_mem_info->num_pages = num_pages; + p_mem_info->num_pages = num_pages; +#ifdef CONFIG_PPC64 + p_mem_info->cache_type = MODS_ALLOC_CACHED; +#else + p_mem_info->cache_type = p->flags & MODS_ALLOC_CACHE_MASK; +#endif + p_mem_info->dma32 = (p->flags & MODS_ALLOC_DMA32) ? true : false; + p_mem_info->contig = (p->flags & MODS_ALLOC_CONTIGUOUS) + ? true : false; + p_mem_info->force_numa = (p->flags & MODS_ALLOC_FORCE_NUMA) + ? true : false; +#ifdef MODS_HASNT_NUMA_NO_NODE p_mem_info->numa_node = numa_node_id(); +#else + p_mem_info->numa_node = NUMA_NO_NODE; +#endif p_mem_info->dev = NULL; + if ((p->flags & MODS_ALLOC_USE_NUMA) && + p->numa_node != MODS_ANY_NUMA_NODE) + p_mem_info->numa_node = p->numa_node; + INIT_LIST_HEAD(&p_mem_info->dma_map_list); - if (p->pci_device.bus <= 0xFFU && p->pci_device.device <= 0xFFU) { - err = mods_find_pci_dev(fp, &p->pci_device, &dev); - if (unlikely(err)) +#ifdef CONFIG_PCI + if (!(p->flags & MODS_ALLOC_USE_NUMA) || + (p->flags & MODS_ALLOC_MAP_DEV)) { + + struct pci_dev *dev = NULL; + + err = mods_find_pci_dev(client, &p->pci_device, &dev); + if (unlikely(err)) { + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); goto failed; + } p_mem_info->dev = dev; -#if defined(MODS_HAS_DEV_TO_NUMA_NODE) - p_mem_info->numa_node = dev_to_node(&dev->dev); -#endif -#if defined(CONFIG_PPC64) && defined(CONFIG_PCI) - if (!mods_is_nvlink_sysmem_trained(fp, dev)) { + if (!(p->flags & MODS_ALLOC_USE_NUMA)) + p_mem_info->numa_node = dev_to_node(&dev->dev); + +#ifdef CONFIG_PPC64 + if (!mods_is_nvlink_sysmem_trained(client, dev)) { /* Until NvLink is trained, we must use memory * on node 0. */ @@ -970,27 +1245,34 @@ int esc_mods_device_alloc_pages_2(struct file *fp, p->pci_device.device, p->pci_device.function, p_mem_info->numa_node); + + if (!(p->flags & MODS_ALLOC_MAP_DEV)) { + pci_dev_put(p_mem_info->dev); + p_mem_info->dev = NULL; + } } +#endif p->memory_handle = 0; - if (p->contiguous) - err = mods_alloc_contig_sys_pages(p_mem_info); + if (p->flags & MODS_ALLOC_CONTIGUOUS) + err = mods_alloc_contig_sys_pages(client, p_mem_info); else { - err = mods_alloc_noncontig_sys_pages(p_mem_info); + err = mods_alloc_noncontig_sys_pages(client, p_mem_info); if (!err) - p_mem_info = optimize_chunks(p_mem_info); + p_mem_info = optimize_chunks(client, p_mem_info); } if (err) { mods_error_printk( - "failed to alloc 0x%x %s bytes, %s, node %d, addrbits %u\n", - p_mem_info->length, - p->contiguous ? "contiguous" : "non-contiguous", + "failed to alloc 0x%llx %s bytes, %s, node %d%s\n", + (unsigned long long)p_mem_info->length, + (p->flags & MODS_ALLOC_CONTIGUOUS) + ? "contiguous" : "non-contiguous", mods_get_prot_str(p_mem_info->cache_type), p_mem_info->numa_node, - (unsigned int)p_mem_info->addr_bits); + p_mem_info->dma32 ? ", dma32" : ""); goto failed; } @@ -998,39 +1280,102 @@ int esc_mods_device_alloc_pages_2(struct file *fp, mods_debug_printk(DEBUG_MEM_DETAILED, "alloc %p\n", p_mem_info); - err = mods_register_alloc(fp, p_mem_info); + err = mods_register_alloc(client, p_mem_info); failed: - if (err) { - if (p_mem_info) { - mods_free_pages(p_mem_info); - pci_dev_put(p_mem_info->dev); - } + if (unlikely(err && p_mem_info)) { + mods_free_pages(client, p_mem_info); + pci_dev_put(p_mem_info->dev); kfree(p_mem_info); + atomic_dec(&client->num_allocs); } LOG_EXT(); return err; } -int esc_mods_device_alloc_pages(struct file *fp, +int esc_mods_device_alloc_pages_2(struct mods_client *client, + struct MODS_DEVICE_ALLOC_PAGES_2 *p) +{ + int err; + u32 flags = 0; + struct MODS_ALLOC_PAGES_2 dev_alloc_pages = {0}; + + LOG_ENT(); + + if (p->contiguous) + flags |= MODS_ALLOC_CONTIGUOUS; + + if (p->address_bits == 32) + flags |= MODS_ALLOC_DMA32; + + if (p->attrib == MODS_MEMORY_UNCACHED) + flags |= MODS_ALLOC_UNCACHED; + else if (p->attrib == MODS_MEMORY_WRITECOMBINE) + flags |= MODS_ALLOC_WRITECOMBINE; + else if (p->attrib != MODS_MEMORY_CACHED) { + mods_error_printk("invalid cache attrib: %u\n", p->attrib); + LOG_EXT(); + return -ENOMEM; + } + + if (p->pci_device.bus > 0xFFU || p->pci_device.device > 0xFFU) + flags |= MODS_ALLOC_USE_NUMA; + else + flags |= MODS_ALLOC_MAP_DEV | MODS_ALLOC_FORCE_NUMA; + + dev_alloc_pages.num_bytes = p->num_bytes; + dev_alloc_pages.flags = flags; + dev_alloc_pages.numa_node = MODS_ANY_NUMA_NODE; + dev_alloc_pages.pci_device = p->pci_device; + + err = esc_mods_alloc_pages_2(client, &dev_alloc_pages); + if (!err) + p->memory_handle = dev_alloc_pages.memory_handle; + + LOG_EXT(); + return err; +} + +int esc_mods_device_alloc_pages(struct mods_client *client, struct MODS_DEVICE_ALLOC_PAGES *p) { int err; - struct MODS_DEVICE_ALLOC_PAGES_2 dev_alloc_pages = {0}; + u32 flags = 0; + struct MODS_ALLOC_PAGES_2 dev_alloc_pages = {0}; LOG_ENT(); - dev_alloc_pages.num_bytes = p->num_bytes; - dev_alloc_pages.contiguous = p->contiguous; - dev_alloc_pages.address_bits = p->address_bits; - dev_alloc_pages.attrib = p->attrib; - dev_alloc_pages.pci_device.domain = 0; - dev_alloc_pages.pci_device.bus = p->pci_device.bus; - dev_alloc_pages.pci_device.device = p->pci_device.device; - dev_alloc_pages.pci_device.function = p->pci_device.function; + if (p->contiguous) + flags |= MODS_ALLOC_CONTIGUOUS; - err = esc_mods_device_alloc_pages_2(fp, &dev_alloc_pages); + if (p->address_bits == 32) + flags |= MODS_ALLOC_DMA32; + + if (p->attrib == MODS_MEMORY_UNCACHED) + flags |= MODS_ALLOC_UNCACHED; + else if (p->attrib == MODS_MEMORY_WRITECOMBINE) + flags |= MODS_ALLOC_WRITECOMBINE; + else if (p->attrib != MODS_MEMORY_CACHED) { + mods_error_printk("invalid cache attrib: %u\n", p->attrib); + LOG_EXT(); + return -ENOMEM; + } + + if (p->pci_device.bus > 0xFFU || p->pci_device.device > 0xFFU) + flags |= MODS_ALLOC_USE_NUMA; + else + flags |= MODS_ALLOC_MAP_DEV | MODS_ALLOC_FORCE_NUMA; + + dev_alloc_pages.num_bytes = p->num_bytes; + dev_alloc_pages.flags = flags; + dev_alloc_pages.numa_node = MODS_ANY_NUMA_NODE; + dev_alloc_pages.pci_device.domain = 0; + dev_alloc_pages.pci_device.bus = p->pci_device.bus; + dev_alloc_pages.pci_device.device = p->pci_device.device; + dev_alloc_pages.pci_device.function = p->pci_device.function; + + err = esc_mods_alloc_pages_2(client, &dev_alloc_pages); if (!err) p->memory_handle = dev_alloc_pages.memory_handle; @@ -1038,23 +1383,39 @@ int esc_mods_device_alloc_pages(struct file *fp, return err; } -int esc_mods_alloc_pages(struct file *fp, struct MODS_ALLOC_PAGES *p) +int esc_mods_alloc_pages(struct mods_client *client, struct MODS_ALLOC_PAGES *p) { int err; - struct MODS_DEVICE_ALLOC_PAGES_2 dev_alloc_pages; + u32 flags = MODS_ALLOC_USE_NUMA; + struct MODS_ALLOC_PAGES_2 dev_alloc_pages = {0}; LOG_ENT(); - dev_alloc_pages.num_bytes = p->num_bytes; - dev_alloc_pages.contiguous = p->contiguous; - dev_alloc_pages.address_bits = p->address_bits; - dev_alloc_pages.attrib = p->attrib; + if (p->contiguous) + flags |= MODS_ALLOC_CONTIGUOUS; + + if (p->address_bits == 32) + flags |= MODS_ALLOC_DMA32; + + if (p->attrib == MODS_MEMORY_UNCACHED) + flags |= MODS_ALLOC_UNCACHED; + else if (p->attrib == MODS_MEMORY_WRITECOMBINE) + flags |= MODS_ALLOC_WRITECOMBINE; + else if (p->attrib != MODS_MEMORY_CACHED) { + mods_error_printk("invalid cache attrib: %u\n", p->attrib); + LOG_EXT(); + return -ENOMEM; + } + + dev_alloc_pages.num_bytes = p->num_bytes; + dev_alloc_pages.flags = flags; + dev_alloc_pages.numa_node = MODS_ANY_NUMA_NODE; dev_alloc_pages.pci_device.domain = 0xFFFFU; - dev_alloc_pages.pci_device.bus = 0xFFFFU; + dev_alloc_pages.pci_device.bus = 0xFFFFU; dev_alloc_pages.pci_device.device = 0xFFFFU; dev_alloc_pages.pci_device.function = 0xFFFFU; - err = esc_mods_device_alloc_pages_2(fp, &dev_alloc_pages); + err = esc_mods_alloc_pages_2(client, &dev_alloc_pages); if (!err) p->memory_handle = dev_alloc_pages.memory_handle; @@ -1062,13 +1423,13 @@ int esc_mods_alloc_pages(struct file *fp, struct MODS_ALLOC_PAGES *p) return err; } -int esc_mods_free_pages(struct file *fp, struct MODS_FREE_PAGES *p) +int esc_mods_free_pages(struct mods_client *client, struct MODS_FREE_PAGES *p) { int err; LOG_ENT(); - err = mods_unregister_and_free(fp, + err = mods_unregister_and_free(client, (struct MODS_MEM_INFO *)(size_t)p->memory_handle); LOG_EXT(); @@ -1076,17 +1437,24 @@ int esc_mods_free_pages(struct file *fp, struct MODS_FREE_PAGES *p) return err; } -int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) +int esc_mods_set_mem_type(struct mods_client *client, + struct MODS_MEMORY_TYPE *p) { struct MODS_MEM_INFO *p_mem_info; - struct mods_client *client = fp->private_data; + u8 type = MODS_ALLOC_CACHED; LOG_ENT(); switch (p->type) { case MODS_MEMORY_CACHED: + break; + case MODS_MEMORY_UNCACHED: + type = MODS_ALLOC_UNCACHED; + break; + case MODS_MEMORY_WRITECOMBINE: + type = MODS_ALLOC_WRITECOMBINE; break; default: @@ -1100,7 +1468,7 @@ int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) return -EINTR; } - p_mem_info = mods_find_alloc(fp, p->physical_address); + p_mem_info = mods_find_alloc(client, p->physical_address); if (p_mem_info) { mutex_unlock(&client->mtx); mods_error_printk("cannot set mem type on phys addr 0x%llx\n", @@ -1111,7 +1479,7 @@ int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) client->mem_type.dma_addr = p->physical_address; client->mem_type.size = p->size; - client->mem_type.type = p->type; + client->mem_type.type = type; mutex_unlock(&client->mtx); @@ -1119,7 +1487,8 @@ int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) return OK; } -int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) +int esc_mods_get_phys_addr(struct mods_client *client, + struct MODS_GET_PHYSICAL_ADDRESS *p) { struct MODS_GET_PHYSICAL_ADDRESS_3 range; int err; @@ -1130,7 +1499,7 @@ int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) range.offset = p->offset; memset(&range.pci_device, 0, sizeof(range.pci_device)); - err = get_addr_range(fp, &range, NULL); + err = get_addr_range(client, &range, NULL); if (!err) p->physical_address = range.physical_address; @@ -1139,7 +1508,7 @@ int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) return err; } -int esc_mods_get_phys_addr_2(struct file *fp, +int esc_mods_get_phys_addr_2(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS_3 *p) { struct MODS_GET_PHYSICAL_ADDRESS_3 range; @@ -1151,7 +1520,7 @@ int esc_mods_get_phys_addr_2(struct file *fp, range.offset = p->offset; memset(&range.pci_device, 0, sizeof(range.pci_device)); - err = get_addr_range(fp, &range, NULL); + err = get_addr_range(client, &range, NULL); if (!err) p->physical_address = range.physical_address; @@ -1160,7 +1529,7 @@ int esc_mods_get_phys_addr_2(struct file *fp, return err; } -int esc_mods_get_mapped_phys_addr(struct file *fp, +int esc_mods_get_mapped_phys_addr(struct mods_client *client, struct MODS_GET_PHYSICAL_ADDRESS *p) { struct MODS_GET_PHYSICAL_ADDRESS_3 range; @@ -1183,10 +1552,10 @@ int esc_mods_get_mapped_phys_addr(struct file *fp, range.pci_device.function = PCI_FUNC(p_mem_info->dev->devfn); - err = get_addr_range(fp, &range, &range.pci_device); + err = get_addr_range(client, &range, &range.pci_device); } else { memset(&range.pci_device, 0, sizeof(range.pci_device)); - err = get_addr_range(fp, &range, NULL); + err = get_addr_range(client, &range, NULL); } if (!err) @@ -1196,8 +1565,8 @@ int esc_mods_get_mapped_phys_addr(struct file *fp, return err; } -int esc_mods_get_mapped_phys_addr_2(struct file *fp, - struct MODS_GET_PHYSICAL_ADDRESS_2 *p) +int esc_mods_get_mapped_phys_addr_2(struct mods_client *client, + struct MODS_GET_PHYSICAL_ADDRESS_2 *p) { struct MODS_GET_PHYSICAL_ADDRESS_3 range; int err; @@ -1208,7 +1577,7 @@ int esc_mods_get_mapped_phys_addr_2(struct file *fp, range.offset = p->offset; range.pci_device = p->pci_device; - err = get_addr_range(fp, &range, &range.pci_device); + err = get_addr_range(client, &range, &range.pci_device); if (!err) p->physical_address = range.physical_address; @@ -1217,8 +1586,8 @@ int esc_mods_get_mapped_phys_addr_2(struct file *fp, return err; } -int esc_mods_get_mapped_phys_addr_3(struct file *fp, - struct MODS_GET_PHYSICAL_ADDRESS_3 *p) +int esc_mods_get_mapped_phys_addr_3(struct mods_client *client, + struct MODS_GET_PHYSICAL_ADDRESS_3 *p) { struct MODS_GET_PHYSICAL_ADDRESS_3 range; int err; @@ -1229,7 +1598,7 @@ int esc_mods_get_mapped_phys_addr_3(struct file *fp, range.offset = p->offset; range.pci_device = p->pci_device; - err = get_addr_range(fp, &range, &range.pci_device); + err = get_addr_range(client, &range, &range.pci_device); if (!err) p->physical_address = range.physical_address; @@ -1238,13 +1607,12 @@ int esc_mods_get_mapped_phys_addr_3(struct file *fp, return err; } -int esc_mods_virtual_to_phys(struct file *fp, +int esc_mods_virtual_to_phys(struct mods_client *client, struct MODS_VIRTUAL_TO_PHYSICAL *p) { struct MODS_GET_PHYSICAL_ADDRESS get_phys_addr; - struct mods_client *client = fp->private_data; - struct list_head *head; - struct list_head *iter; + struct list_head *head; + struct list_head *iter; LOG_ENT(); @@ -1296,7 +1664,7 @@ int esc_mods_virtual_to_phys(struct file *fp, mutex_unlock(&client->mtx); - err = esc_mods_get_phys_addr(fp, &get_phys_addr); + err = esc_mods_get_phys_addr(client, &get_phys_addr); if (err) return err; @@ -1318,11 +1686,10 @@ int esc_mods_virtual_to_phys(struct file *fp, return -EINVAL; } -int esc_mods_phys_to_virtual(struct file *fp, +int esc_mods_phys_to_virtual(struct mods_client *client, struct MODS_PHYSICAL_TO_VIRTUAL *p) { struct SYS_MAP_MEMORY *p_map_mem; - struct mods_client *client = fp->private_data; struct list_head *head; struct list_head *iter; u64 offset; @@ -1394,7 +1761,7 @@ int esc_mods_phys_to_virtual(struct file *fp, return -EINVAL; } -int esc_mods_memory_barrier(struct file *fp) +int esc_mods_memory_barrier(struct mods_client *client) { #if defined(CONFIG_ARM) /* Full memory barrier on ARMv7 */ @@ -1406,7 +1773,7 @@ int esc_mods_memory_barrier(struct file *fp) } #ifdef CONFIG_PCI -int esc_mods_dma_map_memory(struct file *fp, +int esc_mods_dma_map_memory(struct mods_client *client, struct MODS_DMA_MAP_MEMORY *p) { struct MODS_MEM_INFO *p_mem_info; @@ -1442,7 +1809,7 @@ int esc_mods_dma_map_memory(struct file *fp, return 0; } - err = mods_find_pci_dev(fp, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -1454,14 +1821,14 @@ int esc_mods_dma_map_memory(struct file *fp, return err; } - err = mods_create_dma_map(p_mem_info, dev); + err = mods_create_dma_map(client, p_mem_info, dev); pci_dev_put(dev); LOG_EXT(); return err; } -int esc_mods_dma_unmap_memory(struct file *fp, +int esc_mods_dma_unmap_memory(struct mods_client *client, struct MODS_DMA_MAP_MEMORY *p) { struct MODS_MEM_INFO *p_mem_info; @@ -1477,7 +1844,7 @@ int esc_mods_dma_unmap_memory(struct file *fp, return -EINVAL; } - err = mods_find_pci_dev(fp, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -1486,7 +1853,7 @@ int esc_mods_dma_unmap_memory(struct file *fp, p->pci_device.device, p->pci_device.function); } else - err = mods_dma_unmap_all(p_mem_info, dev); + err = mods_dma_unmap_all(client, p_mem_info, dev); pci_dev_put(dev); LOG_EXT(); @@ -1533,7 +1900,7 @@ static void clear_entry_cache_mappings if (!p_mem_info) return; - if (p_mem_info->cache_type != MODS_MEMORY_CACHED) + if (p_mem_info->cache_type != MODS_ALLOC_CACHED) return; for (i = 0; i < p_mem_info->num_chunks; i++) { @@ -1588,12 +1955,11 @@ static void clear_entry_cache_mappings } } -int esc_mods_flush_cpu_cache_range(struct file *fp, +int esc_mods_flush_cpu_cache_range(struct mods_client *client, struct MODS_FLUSH_CPU_CACHE_RANGE *p) { - struct mods_client *client = fp->private_data; - struct list_head *head; - struct list_head *iter; + struct list_head *head; + struct list_head *iter; if (irqs_disabled() || in_interrupt() || p->virt_addr_start > p->virt_addr_end || @@ -1657,27 +2023,31 @@ static int mods_post_alloc(struct MODS_PHYS_CHUNK *chunk, u32 num_pages = 1U << chunk->order; u32 i; - for (i = 0; i < num_pages; i++) { - u64 ptr = 0; - int err = 0; + if (p_mem_info->cache_type != MODS_ALLOC_CACHED && !chunk->wc) { + for (i = 0; i < num_pages; i++) { + u64 ptr = 0; + int err = 0; - ptr = (u64)(size_t)kmap(chunk->p_page + i); - if (!ptr) { - mods_error_printk("kmap failed\n"); - return -EINVAL; - } + ptr = (u64)(size_t)kmap(chunk->p_page + i); + if (!ptr) { + mods_error_printk("kmap failed\n"); + return -EINVAL; + } #if defined(MODS_TEGRA) && !defined(CONFIG_CPA) - clear_contiguous_cache(ptr, - phys_addr + (i << PAGE_SHIFT), - PAGE_SIZE); + clear_contiguous_cache(ptr, + phys_addr + (i << PAGE_SHIFT), + PAGE_SIZE); #else - err = mods_set_mem_type(ptr, 1, p_mem_info->cache_type); + err = mods_set_mem_type(ptr, 1, p_mem_info->cache_type); #endif - kunmap((void *)(size_t)ptr); - if (err) { - mods_error_printk("set cache type failed\n"); - return -EINVAL; + kunmap((void *)(size_t)ptr); + if (err) { + mods_error_printk("set cache type failed\n"); + return -EINVAL; + } } + + chunk->wc = 1; } #ifdef CONFIG_PCI @@ -1703,19 +2073,3 @@ static int mods_post_alloc(struct MODS_PHYS_CHUNK *chunk, return 0; } - -static void mods_pre_free(struct MODS_PHYS_CHUNK *chunk, - struct MODS_MEM_INFO *p_mem_info) -{ - u32 num_pages = 1U << chunk->order; - u32 i; - - for (i = 0; i < num_pages; i++) { - u64 ptr = (u64)(size_t)kmap(chunk->p_page + i); - - if (ptr) - mods_restore_mem_type(ptr, 1, p_mem_info->cache_type); - - kunmap((void *)(size_t)ptr); - } -} diff --git a/drivers/misc/mods/mods_netdevice.c b/drivers/misc/mods/mods_netdevice.c index 8c7c7300..2206246f 100644 --- a/drivers/misc/mods/mods_netdevice.c +++ b/drivers/misc/mods/mods_netdevice.c @@ -1,7 +1,7 @@ /* * mods_net.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2015-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -21,7 +21,8 @@ #include "mods_internal.h" #include -int esc_mods_net_force_link(struct file *pfile, struct MODS_NET_DEVICE_NAME *p) +int esc_mods_net_force_link(struct mods_client *client, + struct MODS_NET_DEVICE_NAME *p) { struct net_device *ndev; diff --git a/drivers/misc/mods/mods_pci.c b/drivers/misc/mods/mods_pci.c index 35365239..55e060f2 100644 --- a/drivers/misc/mods/mods_pci.c +++ b/drivers/misc/mods/mods_pci.c @@ -2,7 +2,7 @@ /* * mods_pci.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -37,13 +37,12 @@ int mods_is_pci_dev(struct pci_dev *dev, dev->devfn == devfn; } -int mods_find_pci_dev(struct file *fp, +int mods_find_pci_dev(struct mods_client *client, struct mods_pci_dev_2 *pcidev, struct pci_dev **retdev) { - struct mods_client *client = fp->private_data; - struct pci_dev *dev; - int err; + struct pci_dev *dev; + int err; if (unlikely(mutex_lock_interruptible(&client->mtx))) return -EINTR; @@ -92,7 +91,7 @@ int mods_find_pci_dev(struct file *fp, return err; } -static int find_pci_dev_impl(struct file *pfile, +static int find_pci_dev_impl(struct mods_client *client, struct MODS_FIND_PCI_DEVICE_2 *p, int enum_non_zero_dom) { @@ -128,13 +127,13 @@ static int find_pci_dev_impl(struct file *pfile, return OK; } -int esc_mods_find_pci_dev_2(struct file *pfile, +int esc_mods_find_pci_dev_2(struct mods_client *client, struct MODS_FIND_PCI_DEVICE_2 *p) { - return find_pci_dev_impl(pfile, p, 1); + return find_pci_dev_impl(client, p, 1); } -int esc_mods_find_pci_dev(struct file *pfile, +int esc_mods_find_pci_dev(struct mods_client *client, struct MODS_FIND_PCI_DEVICE *p) { struct MODS_FIND_PCI_DEVICE_2 p2; @@ -144,7 +143,7 @@ int esc_mods_find_pci_dev(struct file *pfile, p2.vendor_id = p->vendor_id; p2.index = p->index; - err = find_pci_dev_impl(pfile, &p2, 0); + err = find_pci_dev_impl(client, &p2, 0); if (!err) { p->bus_number = p2.pci_device.bus; @@ -155,7 +154,7 @@ int esc_mods_find_pci_dev(struct file *pfile, return err; } -static int mods_find_pci_class_code(struct file *pfile, +static int mods_find_pci_class_code(struct mods_client *client, struct MODS_FIND_PCI_CLASS_CODE_2 *p, int enum_non_zero_dom) { @@ -188,13 +187,13 @@ static int mods_find_pci_class_code(struct file *pfile, return OK; } -int esc_mods_find_pci_class_code_2(struct file *pfile, +int esc_mods_find_pci_class_code_2(struct mods_client *client, struct MODS_FIND_PCI_CLASS_CODE_2 *p) { - return mods_find_pci_class_code(pfile, p, 1); + return mods_find_pci_class_code(client, p, 1); } -int esc_mods_find_pci_class_code(struct file *pfile, +int esc_mods_find_pci_class_code(struct mods_client *client, struct MODS_FIND_PCI_CLASS_CODE *p) { struct MODS_FIND_PCI_CLASS_CODE_2 p2; @@ -203,7 +202,7 @@ int esc_mods_find_pci_class_code(struct file *pfile, p2.class_code = p->class_code; p2.index = p->index; - err = mods_find_pci_class_code(pfile, &p2, 0); + err = mods_find_pci_class_code(client, &p2, 0); if (!err) { p->bus_number = p2.pci_device.bus; @@ -214,7 +213,7 @@ int esc_mods_find_pci_class_code(struct file *pfile, return err; } -int esc_mods_pci_get_bar_info_2(struct file *pfile, +int esc_mods_pci_get_bar_info_2(struct mods_client *client, struct MODS_PCI_GET_BAR_INFO_2 *p) { struct pci_dev *dev; @@ -227,7 +226,7 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, LOG_ENT(); - err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { LOG_EXT(); return err; @@ -249,7 +248,8 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, } /* Enable device on the PCI bus */ - if (mods_enable_device(pfile->private_data, dev) == 0) { + err = mods_enable_device(client, dev, NULL); + if (err) { mods_error_printk("unable to enable dev %04x:%02x:%02x.%x\n", p->pci_device.domain, p->pci_device.bus, @@ -258,7 +258,7 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, mutex_unlock(mods_get_irq_mutex()); pci_dev_put(dev); LOG_EXT(); - return -EINVAL; + return err; } mutex_unlock(mods_get_irq_mutex()); @@ -289,7 +289,7 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, return OK; } -int esc_mods_pci_get_bar_info(struct file *pfile, +int esc_mods_pci_get_bar_info(struct mods_client *client, struct MODS_PCI_GET_BAR_INFO *p) { int err; @@ -301,7 +301,7 @@ int esc_mods_pci_get_bar_info(struct file *pfile, get_bar_info.pci_device.function = p->pci_device.function; get_bar_info.bar_index = p->bar_index; - err = esc_mods_pci_get_bar_info_2(pfile, &get_bar_info); + err = esc_mods_pci_get_bar_info_2(client, &get_bar_info); if (likely(!err)) { p->base_address = get_bar_info.base_address; @@ -311,7 +311,7 @@ int esc_mods_pci_get_bar_info(struct file *pfile, return err; } -int esc_mods_pci_get_irq_2(struct file *pfile, +int esc_mods_pci_get_irq_2(struct mods_client *client, struct MODS_PCI_GET_IRQ_2 *p) { struct pci_dev *dev; @@ -319,7 +319,7 @@ int esc_mods_pci_get_irq_2(struct file *pfile, LOG_ENT(); - err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { LOG_EXT(); return err; @@ -340,7 +340,7 @@ int esc_mods_pci_get_irq_2(struct file *pfile, return OK; } -int esc_mods_pci_get_irq(struct file *pfile, +int esc_mods_pci_get_irq(struct mods_client *client, struct MODS_PCI_GET_IRQ *p) { int err; @@ -351,7 +351,7 @@ int esc_mods_pci_get_irq(struct file *pfile, get_irq.pci_device.device = p->pci_device.device; get_irq.pci_device.function = p->pci_device.function; - err = esc_mods_pci_get_irq_2(pfile, &get_irq); + err = esc_mods_pci_get_irq_2(client, &get_irq); if (likely(!err)) p->irq = get_irq.irq; @@ -359,14 +359,14 @@ int esc_mods_pci_get_irq(struct file *pfile, return err; } -int esc_mods_pci_read_2(struct file *pfile, struct MODS_PCI_READ_2 *p) +int esc_mods_pci_read_2(struct mods_client *client, struct MODS_PCI_READ_2 *p) { struct pci_dev *dev; int err; LOG_ENT(); - err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { LOG_EXT(); return err; @@ -410,7 +410,7 @@ int esc_mods_pci_read_2(struct file *pfile, struct MODS_PCI_READ_2 *p) return err; } -int esc_mods_pci_read(struct file *pfile, struct MODS_PCI_READ *p) +int esc_mods_pci_read(struct mods_client *client, struct MODS_PCI_READ *p) { int err; struct MODS_PCI_READ_2 pci_read = { {0} }; @@ -422,7 +422,7 @@ int esc_mods_pci_read(struct file *pfile, struct MODS_PCI_READ *p) pci_read.address = p->address; pci_read.data_size = p->data_size; - err = esc_mods_pci_read_2(pfile, &pci_read); + err = esc_mods_pci_read_2(client, &pci_read); if (likely(!err)) p->data = pci_read.data; @@ -430,7 +430,7 @@ int esc_mods_pci_read(struct file *pfile, struct MODS_PCI_READ *p) return err; } -int esc_mods_pci_write_2(struct file *pfile, struct MODS_PCI_WRITE_2 *p) +int esc_mods_pci_write_2(struct mods_client *client, struct MODS_PCI_WRITE_2 *p) { struct pci_dev *dev; int err; @@ -447,7 +447,7 @@ int esc_mods_pci_write_2(struct file *pfile, struct MODS_PCI_WRITE_2 *p) p->data_size, p->data); - err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -479,7 +479,7 @@ int esc_mods_pci_write_2(struct file *pfile, struct MODS_PCI_WRITE_2 *p) return err; } -int esc_mods_pci_write(struct file *pfile, +int esc_mods_pci_write(struct mods_client *client, struct MODS_PCI_WRITE *p) { struct MODS_PCI_WRITE_2 pci_write = { {0} }; @@ -492,10 +492,10 @@ int esc_mods_pci_write(struct file *pfile, pci_write.data = p->data; pci_write.data_size = p->data_size; - return esc_mods_pci_write_2(pfile, &pci_write); + return esc_mods_pci_write_2(client, &pci_write); } -int esc_mods_pci_bus_add_dev(struct file *pfile, +int esc_mods_pci_bus_add_dev(struct mods_client *client, struct MODS_PCI_BUS_ADD_DEVICES *scan) { struct pci_bus *bus; @@ -526,7 +526,7 @@ int esc_mods_pci_bus_add_dev(struct file *pfile, return err; } -int esc_mods_pci_hot_reset(struct file *pfile, +int esc_mods_pci_hot_reset(struct mods_client *client, struct MODS_PCI_HOT_RESET *p) { #if defined(CONFIG_PPC64) @@ -542,7 +542,7 @@ int esc_mods_pci_hot_reset(struct file *pfile, p->pci_device.device, p->pci_device.function); - err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("pci_hot_reset cannot find pci device %04x:%02x:%02x.%x\n", @@ -580,11 +580,47 @@ int esc_mods_pci_hot_reset(struct file *pfile, #endif } +int esc_mods_pci_bus_remove_dev(struct mods_client *client, + struct MODS_PCI_BUS_REMOVE_DEV *p) +{ +#if !defined(MODS_HASNT_PCI_BUS_REMOVE_DEV) + struct pci_dev *dev; + int err; + + LOG_ENT(); + + err = mods_find_pci_dev(client, &p->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("pci_remove cannot find pci device %04x:%02x:%02x.%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + LOG_EXT(); + return err; + } + + mods_debug_printk(DEBUG_PCI, + "pci remove on %04x:%02x:%02x.%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + + pci_stop_and_remove_bus_device(dev); + LOG_EXT(); + return err; +#else + return -EINVAL; +#endif +} + /************************ * PIO ESCAPE FUNCTIONS * ************************/ -int esc_mods_pio_read(struct file *pfile, struct MODS_PIO_READ *p) +int esc_mods_pio_read(struct mods_client *client, struct MODS_PIO_READ *p) { LOG_ENT(); switch (p->data_size) { @@ -604,7 +640,7 @@ int esc_mods_pio_read(struct file *pfile, struct MODS_PIO_READ *p) return OK; } -int esc_mods_pio_write(struct file *pfile, struct MODS_PIO_WRITE *p) +int esc_mods_pio_write(struct mods_client *client, struct MODS_PIO_WRITE *p) { LOG_ENT(); switch (p->data_size) { @@ -624,16 +660,15 @@ int esc_mods_pio_write(struct file *pfile, struct MODS_PIO_WRITE *p) return OK; } -int esc_mods_device_numa_info_2(struct file *fp, +int esc_mods_device_numa_info_2(struct mods_client *client, struct MODS_DEVICE_NUMA_INFO_2 *p) { -#ifdef MODS_HAS_DEV_TO_NUMA_NODE struct pci_dev *dev; int err; LOG_ENT(); - err = mods_find_pci_dev(fp, &p->pci_device, &dev); + err = mods_find_pci_dev(client, &p->pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", @@ -674,24 +709,21 @@ int esc_mods_device_numa_info_2(struct file *fp, pci_dev_put(dev); LOG_EXT(); return OK; -#else - return -EINVAL; -#endif } -int esc_mods_device_numa_info(struct file *fp, +int esc_mods_device_numa_info(struct mods_client *client, struct MODS_DEVICE_NUMA_INFO *p) { int err; int i; struct MODS_DEVICE_NUMA_INFO_2 numa_info = { {0} }; - numa_info.pci_device.domain = 0; - numa_info.pci_device.bus = p->pci_device.bus; - numa_info.pci_device.device = p->pci_device.device; - numa_info.pci_device.function = p->pci_device.function; + numa_info.pci_device.domain = 0; + numa_info.pci_device.bus = p->pci_device.bus; + numa_info.pci_device.device = p->pci_device.device; + numa_info.pci_device.function = p->pci_device.function; - err = esc_mods_device_numa_info_2(fp, &numa_info); + err = esc_mods_device_numa_info_2(client, &numa_info); if (likely(!err)) { p->node = numa_info.node; @@ -704,10 +736,10 @@ int esc_mods_device_numa_info(struct file *fp, return err; } -int esc_mods_get_iommu_state(struct file *pfile, +int esc_mods_get_iommu_state(struct mods_client *client, struct MODS_GET_IOMMU_STATE *state) { - int err = esc_mods_get_iommu_state_2(pfile, state); + int err = esc_mods_get_iommu_state_2(client, state); if (!err) state->state = (state->state == MODS_SWIOTLB_DISABLED) ? 1 : 0; @@ -715,7 +747,7 @@ int esc_mods_get_iommu_state(struct file *pfile, return err; } -int esc_mods_get_iommu_state_2(struct file *pfile, +int esc_mods_get_iommu_state_2(struct mods_client *client, struct MODS_GET_IOMMU_STATE *state) { #if !defined(CONFIG_SWIOTLB) @@ -729,7 +761,7 @@ int esc_mods_get_iommu_state_2(struct file *pfile, LOG_ENT(); - err = mods_find_pci_dev(pfile, &state->pci_device, &dev); + err = mods_find_pci_dev(client, &state->pci_device, &dev); if (unlikely(err)) { LOG_EXT(); return err; @@ -758,7 +790,7 @@ int esc_mods_get_iommu_state_2(struct file *pfile, return OK; } -int esc_mods_pci_set_dma_mask(struct file *file, +int esc_mods_pci_set_dma_mask(struct mods_client *client, struct MODS_PCI_DMA_MASK *dma_mask) { int err; @@ -774,7 +806,7 @@ int esc_mods_pci_set_dma_mask(struct file *file, return -EINVAL; } - err = mods_find_pci_dev(file, &dma_mask->pci_device, &dev); + err = mods_find_pci_dev(client, &dma_mask->pci_device, &dev); if (unlikely(err)) { if (err == -ENODEV) mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", diff --git a/drivers/misc/mods/mods_tegradc.c b/drivers/misc/mods/mods_tegradc.c index abf2b445..b6fdad56 100644 --- a/drivers/misc/mods/mods_tegradc.c +++ b/drivers/misc/mods/mods_tegradc.c @@ -1,7 +1,7 @@ /* * mods_tegradc.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2014-2018, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -60,7 +60,7 @@ static void mods_tegra_dc_set_windowattr_basic(struct tegra_dc_win *win, win->out_h); } -int esc_mods_tegra_dc_config_possible(struct file *fp, +int esc_mods_tegra_dc_config_possible(struct mods_client *client, struct MODS_TEGRA_DC_CONFIG_POSSIBLE *args) { int i; diff --git a/drivers/misc/mods/mods_tegraprod.c b/drivers/misc/mods/mods_tegraprod.c index b1e84b49..9ff05228 100644 --- a/drivers/misc/mods/mods_tegraprod.c +++ b/drivers/misc/mods/mods_tegraprod.c @@ -1,7 +1,7 @@ /* * mods_tegraprod.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -66,7 +66,7 @@ int mods_tegra_prod_init(const struct miscdevice *misc_dev) * Returns 0 on success, others for error or no matching node found. */ int esc_mods_tegra_prod_iterate_dt( - struct file *fp, + struct mods_client *client, struct MODS_TEGRA_PROD_ITERATOR *iterator ) { @@ -432,7 +432,7 @@ static int mods_tegra_get_prod_info( * */ int esc_mods_tegra_prod_is_supported( - struct file *fp, + struct mods_client *client, struct MODS_TEGRA_PROD_IS_SUPPORTED *tuple ) { @@ -476,7 +476,7 @@ int esc_mods_tegra_prod_is_supported( * */ int esc_mods_tegra_prod_set_prod_all( - struct file *fp, + struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple ) { @@ -515,7 +515,7 @@ int esc_mods_tegra_prod_set_prod_all( * */ int esc_mods_tegra_prod_set_prod_boot( - struct file *fp, + struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple ) { @@ -555,7 +555,7 @@ int esc_mods_tegra_prod_set_prod_boot( * */ int esc_mods_tegra_prod_set_prod_by_name( - struct file *fp, + struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple ) { @@ -601,7 +601,7 @@ int esc_mods_tegra_prod_set_prod_by_name( * */ int esc_mods_tegra_prod_set_prod_exact( - struct file *fp, + struct mods_client *client, struct MODS_TEGRA_PROD_SET_TUPLE *tuple ) {