diff --git a/drivers/misc/mods/mods.h b/drivers/misc/mods/mods.h index d484cd23..ef25fe8a 100644 --- a/drivers/misc/mods/mods.h +++ b/drivers/misc/mods/mods.h @@ -25,7 +25,7 @@ /* Driver version */ #define MODS_DRIVER_VERSION_MAJOR 3 -#define MODS_DRIVER_VERSION_MINOR 88 +#define MODS_DRIVER_VERSION_MINOR 89 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \ ((MODS_DRIVER_VERSION_MINOR/10) << 4) | \ (MODS_DRIVER_VERSION_MINOR%10)) @@ -125,6 +125,21 @@ struct MODS_GET_PHYSICAL_ADDRESS_3 { __u64 physical_address; }; +#define MAX_PA_ENTRIES 64 + +/* MODS_ESC_GET_PHYSICAL_ADDRESS_RANGE, MODS_ESC_GET_DMA_ADDRESS_RANGE */ +struct MODS_GET_ADDRESS_RANGE { + /* IN */ + __u64 memory_handle; + __u64 offset; + __u32 stride; + __u32 num_entries; + struct mods_pci_dev_2 pci_device; + + /* OUT */ + __u64 physical_addresses[MAX_PA_ENTRIES]; +}; + /* MODS_ESC_VIRTUAL_TO_PHYSICAL */ struct MODS_VIRTUAL_TO_PHYSICAL { /* IN */ @@ -1356,4 +1371,10 @@ struct MODS_MSR { _IOWR(MODS_IOC_MAGIC, 120, struct MODS_MSR) #define MODS_ESC_WRITE_MSR \ _IOW(MODS_IOC_MAGIC, 121, struct MODS_MSR) +#define MODS_ESC_GET_PHYSICAL_ADDRESS_RANGE \ + _IOWR(MODS_IOC_MAGIC, 122, \ + struct MODS_GET_ADDRESS_RANGE) +#define MODS_ESC_GET_DMA_ADDRESS_RANGE \ + _IOWR(MODS_IOC_MAGIC, 123, \ + struct MODS_GET_ADDRESS_RANGE) #endif /* _MODS_H_ */ diff --git a/drivers/misc/mods/mods_acpi.c b/drivers/misc/mods/mods_acpi.c index e2ceb927..bc63a444 100644 --- a/drivers/misc/mods/mods_acpi.c +++ b/drivers/misc/mods/mods_acpi.c @@ -73,54 +73,49 @@ static int mods_extract_acpi_object( u8 *buf_end ) { - int ret = OK; + int err = OK; switch (obj->type) { case ACPI_TYPE_BUFFER: if (obj->buffer.length == 0) { - mods_error_printk( - "empty ACPI output buffer from ACPI method %s\n", - method); - ret = -EINVAL; + mods_error_printk("empty ACPI output buffer from ACPI method %s\n", + method); + err = -EINVAL; } else if (obj->buffer.length <= buf_end-*buf) { u32 size = obj->buffer.length; memcpy(*buf, obj->buffer.pointer, size); *buf += size; } else { - mods_error_printk( - "output buffer too small for ACPI method %s\n", - method); - ret = -EINVAL; + mods_error_printk("output buffer too small for ACPI method %s\n", + method); + err = -EINVAL; } break; case ACPI_TYPE_INTEGER: if (buf_end - *buf >= 4) { if (obj->integer.value > 0xFFFFFFFFU) { - mods_error_printk( - "integer value from ACPI method %s out of range\n", - method); - ret = -EINVAL; + mods_error_printk("integer value from ACPI method %s out of range\n", + method); + err = -EINVAL; } else { memcpy(*buf, &obj->integer.value, 4); *buf += 4; } } else { - mods_error_printk( - "output buffer too small for ACPI method %s\n", - method); - ret = -EINVAL; + mods_error_printk("output buffer too small for ACPI method %s\n", + method); + err = -EINVAL; } break; case ACPI_TYPE_PACKAGE: if (obj->package.count == 0) { - mods_error_printk( - "empty ACPI output package from ACPI method %s\n", - method); - ret = -EINVAL; + mods_error_printk("empty ACPI output package from ACPI method %s\n", + method); + err = -EINVAL; } else { union acpi_object *elements = obj->package.elements; u32 size = 0; @@ -128,44 +123,43 @@ static int mods_extract_acpi_object( for (i = 0; i < obj->package.count; i++) { u8 *old_buf = *buf; + u32 new_size; - ret = mods_extract_acpi_object(method, + err = mods_extract_acpi_object(method, &elements[i], buf, buf_end); - if (ret == OK) { - u32 new_size = *buf - old_buf; - - if (size == 0) { - size = new_size; - } else if (size != new_size) { - mods_error_printk( - "ambiguous package element size from ACPI method %s\n", - method); - ret = -EINVAL; - } - } else + if (err) break; + + new_size = *buf - old_buf; + + if (size == 0) { + size = new_size; + } else if (size != new_size) { + mods_error_printk("ambiguous package element size from ACPI method %s\n", + method); + err = -EINVAL; + } } } break; default: - mods_error_printk( - "unsupported ACPI output type 0x%02x from method %s\n", - (unsigned int)obj->type, method); - ret = -EINVAL; + mods_error_printk("unsupported ACPI output type 0x%02x from method %s\n", + (unsigned int)obj->type, method); + err = -EINVAL; break; } - return ret; + return err; } static int mods_eval_acpi_method(struct file *pfile, struct MODS_EVAL_ACPI_METHOD *p, struct mods_pci_dev_2 *pdevice) { - int ret = OK; + int err = OK; int i; acpi_status status; struct acpi_object_list input; @@ -173,24 +167,35 @@ static int mods_eval_acpi_method(struct file *pfile, union acpi_object *acpi_method = NULL; union acpi_object acpi_params[ACPI_MAX_ARGUMENT_NUMBER]; acpi_handle acpi_method_handler = NULL; + struct pci_dev *dev = NULL; + + LOG_ENT(); + + if (p->argument_count >= ACPI_MAX_ARGUMENT_NUMBER) { + mods_error_printk("invalid argument count for ACPI call\n"); + LOG_EXT(); + return -EINVAL; + } if (pdevice) { - unsigned int devfn; - struct pci_dev *dev; - mods_debug_printk(DEBUG_ACPI, "ACPI %s for device %04x:%02x:%02x.%x\n", p->method_name, - (unsigned int)pdevice->domain, - (unsigned int)pdevice->bus, - (unsigned int)pdevice->device, - (unsigned int)pdevice->function); + pdevice->domain, + pdevice->bus, + pdevice->device, + pdevice->function); - devfn = PCI_DEVFN(pdevice->device, pdevice->function); - dev = MODS_PCI_GET_SLOT(pdevice->domain, pdevice->bus, devfn); - if (!dev) { - mods_error_printk("ACPI: PCI device not found\n"); - return -EINVAL; + err = mods_find_pci_dev(pfile, pdevice, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("ACPI: PCI device %04x:%02x:%02x.%x not found\n", + pdevice->domain, + pdevice->bus, + pdevice->device, + pdevice->function); + LOG_EXT(); + return err; } acpi_method_handler = MODS_ACPI_HANDLE(&dev->dev); } else { @@ -201,11 +206,8 @@ static int mods_eval_acpi_method(struct file *pfile, if (!acpi_method_handler) { mods_debug_printk(DEBUG_ACPI, "ACPI: handle for %s not found\n", p->method_name); - return -EINVAL; - } - - if (p->argument_count >= ACPI_MAX_ARGUMENT_NUMBER) { - mods_error_printk("invalid argument count for ACPI call\n"); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } @@ -227,6 +229,8 @@ static int mods_eval_acpi_method(struct file *pfile, } default: { mods_error_printk("unsupported ACPI argument type\n"); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } } @@ -242,6 +246,8 @@ static int mods_eval_acpi_method(struct file *pfile, if (ACPI_FAILURE(status)) { mods_error_printk("ACPI method %s failed\n", p->method_name); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } @@ -249,63 +255,76 @@ static int mods_eval_acpi_method(struct file *pfile, if (!acpi_method) { mods_error_printk("missing output from ACPI method %s\n", p->method_name); - ret = -EINVAL; + err = -EINVAL; } else { u8 *buf = p->out_buffer; - ret = mods_extract_acpi_object(p->method_name, + err = mods_extract_acpi_object(p->method_name, acpi_method, &buf, buf+sizeof(p->out_buffer)); - p->out_data_size = (ret == OK) ? (buf - p->out_buffer) : 0; + p->out_data_size = err ? 0 : (buf - p->out_buffer); } kfree(output.pointer); - return ret; + pci_dev_put(dev); + LOG_EXT(); + return err; } static int mods_acpi_get_ddc(struct file *pfile, struct MODS_ACPI_GET_DDC_2 *p, struct mods_pci_dev_2 *pci_device) { - acpi_status status; - struct acpi_device *device = NULL; - struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; - union acpi_object *ddc; - union acpi_object ddc_arg0 = { ACPI_TYPE_INTEGER }; - struct acpi_object_list input = { 1, &ddc_arg0 }; - struct list_head *node, *next; - u32 i; - acpi_handle dev_handle = NULL; - acpi_handle lcd_dev_handle = NULL; + int err; + acpi_status status; + struct acpi_device *device = NULL; + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *ddc; + union acpi_object ddc_arg0 = { ACPI_TYPE_INTEGER }; + struct acpi_object_list input = { 1, &ddc_arg0 }; + struct list_head *node; + struct list_head *next; + u32 i; + acpi_handle dev_handle = NULL; + acpi_handle lcd_dev_handle = NULL; + struct pci_dev *dev = NULL; + + LOG_ENT(); mods_debug_printk(DEBUG_ACPI, "ACPI _DDC (EDID) for device %04x:%02x:%02x.%x\n", - (unsigned int)pci_device->domain, - (unsigned int)pci_device->bus, - (unsigned int)pci_device->device, - (unsigned int)pci_device->function); + pci_device->domain, + pci_device->bus, + pci_device->device, + pci_device->function); - { - unsigned int devfn = PCI_DEVFN(pci_device->device, - pci_device->function); - struct pci_dev *dev = MODS_PCI_GET_SLOT(pci_device->domain, - pci_device->bus, devfn); - if (!dev) { - mods_error_printk("ACPI: PCI device not found\n"); - return -EINVAL; - } - dev_handle = MODS_ACPI_HANDLE(&dev->dev); + err = mods_find_pci_dev(pfile, pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("ACPI: PCI device %04x:%02x:%02x.%x not found\n", + pci_device->domain, + pci_device->bus, + pci_device->device, + pci_device->function); + LOG_EXT(); + return err; } + + dev_handle = MODS_ACPI_HANDLE(&dev->dev); if (!dev_handle) { mods_debug_printk(DEBUG_ACPI, "ACPI: handle for _DDC not found\n"); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } - status = acpi_bus_get_device(dev_handle, &device); + status = acpi_bus_get_device(dev_handle, &device); if (ACPI_FAILURE(status) || !device) { mods_error_printk("ACPI: device for _DDC not found\n"); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } @@ -341,10 +360,10 @@ static int mods_acpi_get_ddc(struct file *pfile, mods_debug_printk(DEBUG_ACPI, "ACPI: Found LCD 0x%x on device %04x:%02x:%02x.%x\n", (unsigned int)device_id, - (unsigned int)p->device.domain, - (unsigned int)p->device.bus, - (unsigned int)p->device.device, - (unsigned int)p->device.function); + p->device.domain, + p->device.bus, + p->device.device, + p->device.function); break; } @@ -353,10 +372,12 @@ static int mods_acpi_get_ddc(struct file *pfile, if (lcd_dev_handle == NULL) { mods_error_printk( "ACPI: LCD not found for device %04x:%02x:%02x.%x\n", - (unsigned int)p->device.domain, - (unsigned int)p->device.bus, - (unsigned int)p->device.device, - (unsigned int)p->device.function); + p->device.domain, + p->device.bus, + p->device.device, + p->device.function); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } @@ -377,6 +398,8 @@ static int mods_acpi_get_ddc(struct file *pfile, if (ACPI_FAILURE(status)) { mods_error_printk("ACPI method _DDC (EDID) failed\n"); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } @@ -389,20 +412,21 @@ static int mods_acpi_get_ddc(struct file *pfile, memcpy(p->out_buffer, ddc->buffer.pointer, p->out_data_size); + err = OK; } else { mods_error_printk( "output buffer too small for ACPI method _DDC (EDID)\n"); - kfree(output.pointer); - return -EINVAL; + err = -EINVAL; } } else { mods_error_printk("unsupported ACPI output type\n"); - kfree(output.pointer); - return -EINVAL; + err = -EINVAL; } kfree(output.pointer); - return OK; + pci_dev_put(dev); + LOG_EXT(); + return err; } /************************* diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h index e1d358a1..de44bfcf 100644 --- a/drivers/misc/mods/mods_internal.h +++ b/drivers/misc/mods/mods_internal.h @@ -97,6 +97,9 @@ struct mods_client { spinlock_t irq_lock; struct en_dev_entry *enabled_devices; struct mem_type mem_type; +#if defined(CONFIG_PCI) + struct pci_dev *cached_dev; +#endif struct mutex mtx; int mods_fb_suspended[FB_MAX]; u32 access_token; @@ -121,34 +124,35 @@ struct MODS_PHYS_CHUNK { u64 dma_addr:58; /* phys addr (or machine addr on XEN) */ u32 order:5; /* 1<bus) == mydomain \ - && __dev->bus->number == mybus \ - && __dev->devfn == devfn) \ - break; \ - } \ - __dev; \ -}) - /* ACPI */ #ifdef MODS_HAS_NEW_ACPI_WALK #define MODS_ACPI_WALK_NAMESPACE(type, start_object, max_depth, user_function, \ @@ -398,12 +391,24 @@ int mods_unregister_all_ppc_tce_bypass(struct file *fp); int mods_unregister_all_nvlink_sysmem_trained(struct file *fp); #endif +/* pci */ #ifdef CONFIG_PCI struct en_dev_entry *mods_enable_device(struct mods_client *client, struct pci_dev *dev); void mods_disable_device(struct pci_dev *pdev); #endif +#ifdef CONFIG_PCI +int mods_is_pci_dev(struct pci_dev *dev, + struct mods_pci_dev_2 *pcidev); +int mods_find_pci_dev(struct file *fp, + struct mods_pci_dev_2 *pcidev, + struct pci_dev **retdev); +#else +#define mods_is_pci_dev(a, b) 0 +#define mods_find_pci_dev(a, b, c) (-ENODEV) +#endif + /* clock */ #ifdef MODS_TEGRA void mods_init_clock_api(void); @@ -424,12 +429,16 @@ int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p); int esc_mods_get_phys_addr_2(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_3 *p); +int esc_mods_get_phys_addr_range(struct file *fp, + struct MODS_GET_ADDRESS_RANGE *p); int esc_mods_get_mapped_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p); int esc_mods_get_mapped_phys_addr_2(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_2 *p); int esc_mods_get_mapped_phys_addr_3(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_3 *p); +int esc_mods_get_dma_addr_range(struct file *fp, + struct MODS_GET_ADDRESS_RANGE *p); int esc_mods_virtual_to_phys(struct file *fp, struct MODS_VIRTUAL_TO_PHYSICAL *p); int esc_mods_phys_to_virtual(struct file *fp, diff --git a/drivers/misc/mods/mods_irq.c b/drivers/misc/mods/mods_irq.c index 410e85e4..6435c79c 100644 --- a/drivers/misc/mods/mods_irq.c +++ b/drivers/misc/mods/mods_irq.c @@ -59,30 +59,18 @@ struct mutex *mods_get_irq_mutex(void) struct en_dev_entry *mods_enable_device(struct mods_client *client, struct pci_dev *dev) { - int ret = -1; + int err = OK; struct en_dev_entry *dpriv = client->enabled_devices; BUG_ON(!mutex_is_locked(&mp.mtx)); dpriv = pci_get_drvdata(dev); - - if (dpriv) { + if (unlikely(dpriv)) { if (dpriv->client_id == client->client_id) return dpriv; mods_error_printk("invalid client %u for device %04x:%02x:%02x.%x\n", - (unsigned int)client->client_id, - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn)); - return 0; - } - - ret = pci_enable_device(dev); - - if (ret != 0) { - mods_error_printk("failed to enable device %04x:%02x:%02x.%x\n", + client->client_id, pci_domain_nr(dev->bus), dev->bus->number, PCI_SLOT(dev->devfn), @@ -93,9 +81,21 @@ struct en_dev_entry *mods_enable_device(struct mods_client *client, dpriv = kzalloc(sizeof(*dpriv), GFP_KERNEL | __GFP_NORETRY); if (unlikely(!dpriv)) return 0; - dpriv->client_id = client->client_id; - dpriv->dev = dev; - dpriv->next = client->enabled_devices; + + err = pci_enable_device(dev); + if (unlikely(err)) { + mods_error_printk("failed to enable device %04x:%02x:%02x.%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + kfree(dpriv); + return 0; + } + + dpriv->client_id = client->client_id; + dpriv->dev = pci_dev_get(dev); + dpriv->next = client->enabled_devices; client->enabled_devices = dpriv; pci_set_drvdata(dev, dpriv); @@ -108,8 +108,10 @@ void mods_disable_device(struct pci_dev *dev) BUG_ON(!mutex_is_locked(&mp.mtx)); - if (dpriv) + if (dpriv) { pci_set_drvdata(dev, NULL); + pci_dev_put(dev); + } pci_disable_device(dev); } @@ -245,14 +247,14 @@ static int rec_irq_done(struct dev_irq_map *t, #ifdef CONFIG_PCI if (t->dev) { mods_debug_printk(DEBUG_ISR_DETAILED, - "%04x:%02x:%02x.%x %s IRQ 0x%x time=%uus\n", - (unsigned int)(pci_domain_nr(t->dev->bus)), - (unsigned int)(t->dev->bus->number), - (unsigned int)PCI_SLOT(t->dev->devfn), - (unsigned int)PCI_FUNC(t->dev->devfn), - mods_irq_type_name(t->type), - t->apic_irq, - irq_time); + "%04x:%02x:%02x.%x %s IRQ 0x%x time=%uus\n", + pci_domain_nr(t->dev->bus), + t->dev->bus->number, + PCI_SLOT(t->dev->devfn), + PCI_FUNC(t->dev->devfn), + mods_irq_type_name(t->type), + t->apic_irq, + irq_time); } else #endif mods_debug_printk(DEBUG_ISR_DETAILED, @@ -363,13 +365,13 @@ static int is_nvidia_gpu(struct pci_dev *dev) #ifdef CONFIG_PCI static void setup_mask_info(struct dev_irq_map *newmap, struct MODS_REGISTER_IRQ_4 *p, - struct pci_dev *pdev) + struct pci_dev *dev) { /* account for legacy adapters */ char *bar = newmap->dev_irq_aperture; u32 ii = 0; - if ((p->mask_info_cnt == 0) && is_nvidia_gpu(pdev)) { + if ((p->mask_info_cnt == 0) && is_nvidia_gpu(dev)) { newmap->mask_info_cnt = 1; newmap->mask_info[0].dev_irq_mask_reg = (u32 *)(bar+0x140); newmap->mask_info[0].dev_irq_disable_reg = (u32 *)(bar+0x140); @@ -394,12 +396,14 @@ static void setup_mask_info(struct dev_irq_map *newmap, } #endif -static int add_irq_map(u8 client_id, - struct pci_dev *pdev, - struct MODS_REGISTER_IRQ_4 *p, u32 irq, u32 entry) +static int add_irq_map(u8 client_id, + struct pci_dev *dev, + struct MODS_REGISTER_IRQ_4 *p, + u32 irq, + u32 entry) { - u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); - struct dev_irq_map *newmap = NULL; + u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); + struct dev_irq_map *newmap = NULL; LOG_ENT(); @@ -412,7 +416,7 @@ static int add_irq_map(u8 client_id, /* Fill out the new entry */ newmap->apic_irq = irq; - newmap->dev = pdev; + newmap->dev = dev; newmap->client_id = client_id; newmap->dev_irq_aperture = 0; newmap->mask_info_cnt = 0; @@ -451,8 +455,11 @@ static int add_irq_map(u8 client_id, } newmap->dev_irq_aperture = bar; - setup_mask_info(newmap, p, pdev); + setup_mask_info(newmap, p, dev); } + + if (dev) + pci_dev_get(dev); #endif /* Print out successful registration string */ @@ -463,46 +470,46 @@ static int add_irq_map(u8 client_id, (irq_type == MODS_IRQ_TYPE_MSI) || (irq_type == MODS_IRQ_TYPE_MSIX)) { mods_debug_printk(DEBUG_ISR, - "%04x:%02x:%02x.%x registered %s IRQ 0x%x\n", - (unsigned int)(pci_domain_nr(pdev->bus)), - (unsigned int)(pdev->bus->number), - (unsigned int)PCI_SLOT(pdev->devfn), - (unsigned int)PCI_FUNC(pdev->devfn), - mods_irq_type_name(irq_type), - irq); + "%04x:%02x:%02x.%x registered %s IRQ 0x%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + mods_irq_type_name(irq_type), + irq); } #endif #ifdef CONFIG_PCI_MSI else if (irq_type == MODS_IRQ_TYPE_MSI) { u16 control; u16 data; - int cap_pos = pci_find_capability(pdev, PCI_CAP_ID_MSI); + int cap_pos = pci_find_capability(dev, PCI_CAP_ID_MSI); - pci_read_config_word(pdev, MSI_CONTROL_REG(cap_pos), &control); + pci_read_config_word(dev, MSI_CONTROL_REG(cap_pos), &control); if (IS_64BIT_ADDRESS(control)) - pci_read_config_word(pdev, + pci_read_config_word(dev, MSI_DATA_REG(cap_pos, 1), &data); else - pci_read_config_word(pdev, + pci_read_config_word(dev, MSI_DATA_REG(cap_pos, 0), &data); mods_debug_printk(DEBUG_ISR, - "%04x:%02x:%02x.%x registered MSI IRQ 0x%x data:0x%02x\n", - (unsigned int)(pci_domain_nr(pdev->bus)), - (unsigned int)(pdev->bus->number), - (unsigned int)PCI_SLOT(pdev->devfn), - (unsigned int)PCI_FUNC(pdev->devfn), - irq, - (unsigned int)data); + "%04x:%02x:%02x.%x registered MSI IRQ 0x%x data:0x%02x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + irq, + data); } else if (irq_type == MODS_IRQ_TYPE_MSIX) { mods_debug_printk(DEBUG_ISR, - "%04x:%02x:%02x.%x registered MSI-X IRQ 0x%x\n", - (unsigned int)(pci_domain_nr(pdev->bus)), - (unsigned int)(pdev->bus->number), - (unsigned int)PCI_SLOT(pdev->devfn), - (unsigned int)PCI_FUNC(pdev->devfn), - irq); + "%04x:%02x:%02x.%x registered MSI-X IRQ 0x%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + irq); } #endif @@ -529,6 +536,10 @@ static void mods_free_map(struct dev_irq_map *del) if (del->dev_irq_aperture) iounmap(del->dev_irq_aperture); +#ifdef CONFIG_PCI + pci_dev_put(del->dev); +#endif + /* Free memory */ kfree(del); @@ -641,7 +652,7 @@ static int mods_free_irqs(u8 client_id, struct pci_dev *dev) if (dpriv->client_id != client_id) { mods_error_printk("invalid client %u for device %04x:%02x:%02x.%x\n", - (unsigned int)client_id, + client_id, pci_domain_nr(dev->bus), dev->bus->number, PCI_SLOT(dev->devfn), @@ -652,14 +663,13 @@ static int mods_free_irqs(u8 client_id, struct pci_dev *dev) } mods_debug_printk(DEBUG_ISR_DETAILED, - "(dev=%04x:%02x:%02x.%x) irq_flags=0x%x nvecs=%d\n", - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn), - dpriv->irq_flags, - dpriv->nvecs - ); + "(dev=%04x:%02x:%02x.%x) irq_flags=0x%x nvecs=%d\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + dpriv->irq_flags, + dpriv->nvecs); /* Delete device interrupts from the list */ list_for_each_entry_safe(del, next, &client->irq_list, list) { @@ -668,13 +678,13 @@ static int mods_free_irqs(u8 client_id, struct pci_dev *dev) list_del(&del->list); mods_debug_printk(DEBUG_ISR, - "%04x:%02x:%02x.%x unregistered %s IRQ 0x%x\n", - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn), - mods_irq_type_name(type), - del->apic_irq); + "%04x:%02x:%02x.%x unregistered %s IRQ 0x%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + mods_irq_type_name(type), + del->apic_irq); mods_free_map(del); BUG_ON(type != @@ -748,24 +758,23 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, LOG_ENT(); mods_debug_printk(DEBUG_ISR_DETAILED, - "(dev=%04x:%02x:%02x.%x, flags=0x%x, nvecs=%d)\n", - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn), - flags, - nvecs); + "(dev=%04x:%02x:%02x.%x, flags=0x%x, nvecs=%d)\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + flags, + nvecs); /* Determine if the device supports requested interrupt type */ if (irq_type == MODS_IRQ_TYPE_MSI) { #ifdef CONFIG_PCI_MSI if (pci_find_capability(dev, PCI_CAP_ID_MSI) == 0) { - mods_error_printk( - "dev %04x:%02x:%02x.%x does not support MSI\n", - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn)); + mods_error_printk("dev %04x:%02x:%02x.%x does not support MSI\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); LOG_EXT(); return -EINVAL; } @@ -776,12 +785,11 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, } else if (irq_type == MODS_IRQ_TYPE_MSIX) { #ifdef CONFIG_PCI_MSI if (pci_find_capability(dev, PCI_CAP_ID_MSIX) == 0) { - mods_error_printk( - "dev %04x:%02x:%02x.%x does not support MSI-X\n", - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn)); + mods_error_printk("dev %04x:%02x:%02x.%x does not support MSI-X\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); LOG_EXT(); return -EINVAL; } @@ -818,12 +826,11 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, return -EINVAL; } if (pci_enable_msi(dev) != 0) { - mods_error_printk( - "unable to enable MSI on dev %04x:%02x:%02x.%x\n", - pci_domain_nr(dev->bus), - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn)); + mods_error_printk("unable to enable MSI on dev %04x:%02x:%02x.%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); LOG_EXT(); return -EINVAL; } @@ -911,45 +918,44 @@ static int mods_allocate_irqs(u8 client_id, struct file *pfile, static int mods_register_pci_irq(struct file *pfile, struct MODS_REGISTER_IRQ_4 *p) { - int rc = OK; - unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); - struct pci_dev *dev; + int err = OK; + unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); + struct pci_dev *dev; struct en_dev_entry *dpriv; - unsigned int devfn; - u8 client_id; - int i; + u8 client_id; + int i; LOG_ENT(); /* Identify the caller */ client_id = get_client_id(pfile); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (unlikely(!is_client_id_valid(client_id))) { + WARN_ON(!is_client_id_valid(client_id)); LOG_EXT(); return -EINVAL; } - /* Get the PCI device structure for the specified device from kernel */ - devfn = PCI_DEVFN(p->dev.device, p->dev.function); - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { - mods_error_printk( - "unknown dev %04x:%02x:%02x.%x\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); - LOG_EXT(); - return -EINVAL; - } - - if (!p->irq_count) { + if (unlikely(!p->irq_count)) { mods_error_printk("no irq's requested!\n"); LOG_EXT(); return -EINVAL; } + /* Get the PCI device structure for the specified device from kernel */ + err = mods_find_pci_dev(pfile, &p->dev, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->dev.domain, + p->dev.bus, + p->dev.device, + p->dev.function); + LOG_EXT(); + return err; + } + if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + pci_dev_put(dev); LOG_EXT(); return -EINTR; } @@ -958,22 +964,24 @@ static int mods_register_pci_irq(struct file *pfile, if (dpriv) { if (dpriv->client_id != client_id) { mods_error_printk("dev %04x:%02x:%02x.%x already owned by client %u\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function, - (unsigned int)dpriv->client_id); + p->dev.domain, + p->dev.bus, + p->dev.device, + p->dev.function, + dpriv->client_id); mutex_unlock(&mp.mtx); + pci_dev_put(dev); LOG_EXT(); return -EINVAL; } if (dpriv->nvecs) { mods_error_printk("interrupt for dev %04x:%02x:%02x.%x already registered\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); + p->dev.domain, + p->dev.bus, + p->dev.device, + p->dev.function); mutex_unlock(&mp.mtx); + pci_dev_put(dev); LOG_EXT(); return -EINVAL; } @@ -984,6 +992,7 @@ static int mods_register_pci_irq(struct file *pfile, mods_error_printk("could not allocate irqs for irq_type %d\n", irq_type); mutex_unlock(&mp.mtx); + pci_dev_put(dev); LOG_EXT(); return -EINVAL; } @@ -995,37 +1004,38 @@ static int mods_register_pci_irq(struct file *pfile, (irq_type == MODS_IRQ_TYPE_MSI)) ? dev->irq : dpriv->msix_entries[i].vector; - if (add_irq_map(client_id, dev, p, irq, i) != OK) { + err = add_irq_map(client_id, dev, p, irq, i); + if (unlikely(err)) { #ifdef CONFIG_PCI_MSI if (irq_type == MODS_IRQ_TYPE_MSI) pci_disable_msi(dev); else if (irq_type == MODS_IRQ_TYPE_MSIX) pci_disable_msix(dev); #endif - mutex_unlock(&mp.mtx); - LOG_EXT(); - return -EINVAL; + break; } } mutex_unlock(&mp.mtx); + pci_dev_put(dev); LOG_EXT(); - return rc; + return err; } #endif /* CONFIG_PCI */ static int mods_register_cpu_irq(struct file *pfile, struct MODS_REGISTER_IRQ_4 *p) { - u8 client_id; + u8 client_id; u32 irq = p->dev.bus; + int err; LOG_ENT(); /* Identify the caller */ client_id = get_client_id(pfile); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (unlikely(!is_client_id_valid(client_id))) { + WARN_ON(!is_client_id_valid(client_id)); LOG_EXT(); return -EINVAL; } @@ -1045,14 +1055,11 @@ static int mods_register_cpu_irq(struct file *pfile, } /* Register interrupt */ - if (add_irq_map(client_id, 0, p, irq, 0) != OK) { - mutex_unlock(&mp.mtx); - LOG_EXT(); - return -EINVAL; - } + err = add_irq_map(client_id, 0, p, irq, 0); mutex_unlock(&mp.mtx); - return OK; + LOG_EXT(); + return err; } #ifdef CONFIG_PCI @@ -1060,32 +1067,31 @@ static int mods_unregister_pci_irq(struct file *pfile, struct MODS_REGISTER_IRQ_2 *p) { struct pci_dev *dev; - unsigned int devfn; - u8 client_id; - int rv = OK; + u8 client_id; + int err = OK; LOG_ENT(); /* Identify the caller */ client_id = get_client_id(pfile); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (unlikely(!is_client_id_valid(client_id))) { + WARN_ON(!is_client_id_valid(client_id)); LOG_EXT(); return -EINVAL; } /* Get the PCI device structure for the specified device from kernel */ - devfn = PCI_DEVFN(p->dev.device, p->dev.function); - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { + err = mods_find_pci_dev(pfile, &p->dev, &dev); + if (unlikely(err)) { LOG_EXT(); - return -EINVAL; + return err; } - rv = mods_free_irqs(client_id, dev); + err = mods_free_irqs(client_id, dev); + pci_dev_put(dev); LOG_EXT(); - return rv; + return err; } #endif @@ -1104,8 +1110,8 @@ static int mods_unregister_cpu_irq(struct file *pfile, /* Identify the caller */ client_id = get_client_id(pfile); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (unlikely(!is_client_id_valid(client_id))) { + WARN_ON(!is_client_id_valid(client_id)); LOG_EXT(); return -EINVAL; } @@ -1211,17 +1217,17 @@ int esc_mods_register_irq_2(struct file *pfile, #ifdef CONFIG_PCI { /* Get the PCI device structure */ - unsigned int devfn; struct pci_dev *dev; + int err; + + err = mods_find_pci_dev(pfile, &p->dev, &dev); + if (unlikely(err)) + return err; - devfn = PCI_DEVFN(p->dev.device, p->dev.function); - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { - LOG_EXT(); - return -EINVAL; - } irq_data.aperture_addr = pci_resource_start(dev, 0); irq_data.aperture_size = pci_resource_len(dev, 0); + + pci_dev_put(dev); } #endif @@ -1243,7 +1249,7 @@ int esc_mods_register_irq(struct file *pfile, } int esc_mods_unregister_irq_2(struct file *pfile, - struct MODS_REGISTER_IRQ_2 *p) + struct MODS_REGISTER_IRQ_2 *p) { if (p->type == MODS_IRQ_TYPE_CPU) return mods_unregister_cpu_irq(pfile, p); @@ -1255,7 +1261,7 @@ int esc_mods_unregister_irq_2(struct file *pfile, } int esc_mods_unregister_irq(struct file *pfile, - struct MODS_REGISTER_IRQ *p) + struct MODS_REGISTER_IRQ *p) { struct MODS_REGISTER_IRQ_2 register_irq = { {0} }; @@ -1281,8 +1287,8 @@ int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) /* Identify the caller */ client_id = get_client_id(pfile); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (unlikely(!is_client_id_valid(client_id))) { + WARN_ON(!is_client_id_valid(client_id)); LOG_EXT(); return -EINVAL; } @@ -1320,14 +1326,14 @@ int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) /* Print info about IRQ status returned */ if (dev) { mods_debug_printk(DEBUG_ISR_DETAILED, - "retrieved IRQ index=%d dev %04x:%02x:%02x.%x, time=%uus, delay=%uus\n", - p->irq_list[i].irq_index, - (unsigned int)p->irq_list[i].dev.domain, - (unsigned int)p->irq_list[i].dev.bus, - (unsigned int)p->irq_list[i].dev.device, - (unsigned int)p->irq_list[i].dev.function, - q->data[index].time, - p->irq_list[i].delay); + "retrieved IRQ index=%d dev %04x:%02x:%02x.%x, time=%uus, delay=%uus\n", + p->irq_list[i].irq_index, + p->irq_list[i].dev.domain, + p->irq_list[i].dev.bus, + p->irq_list[i].dev.device, + p->irq_list[i].dev.function, + q->data[index].time, + p->irq_list[i].delay); } else { mods_debug_printk(DEBUG_ISR_DETAILED, "retrieved IRQ 0x%x, time=%uus, delay=%uus\n", @@ -1366,7 +1372,7 @@ int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) } int esc_mods_query_irq(struct file *pfile, - struct MODS_QUERY_IRQ *p) + struct MODS_QUERY_IRQ *p) { int retval, i; struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; @@ -1395,7 +1401,7 @@ int esc_mods_irq_handled_2(struct file *pfile, u32 irq = p->dev.bus; struct dev_irq_map *t = NULL; struct dev_irq_map *next = NULL; - int ret = -EINVAL; + int err = -EINVAL; if (p->type != MODS_IRQ_TYPE_CPU) return -EINVAL; @@ -1404,8 +1410,8 @@ int esc_mods_irq_handled_2(struct file *pfile, /* Identify the caller */ client_id = get_client_id(pfile); - WARN_ON(!is_client_id_valid(client_id)); - if (!is_client_id_valid(client_id)) { + if (unlikely(!is_client_id_valid(client_id))) { + WARN_ON(!is_client_id_valid(client_id)); LOG_EXT(); return -EINVAL; } @@ -1425,7 +1431,7 @@ int esc_mods_irq_handled_2(struct file *pfile, "IRQ type doesn't match registered IRQ\n"); } else { enable_irq(irq); - ret = OK; + err = OK; } break; } @@ -1435,7 +1441,7 @@ int esc_mods_irq_handled_2(struct file *pfile, spin_unlock_irqrestore(&client->irq_lock, flags); LOG_EXT(); - return ret; + return err; } int esc_mods_irq_handled(struct file *pfile, @@ -1454,9 +1460,9 @@ int esc_mods_irq_handled(struct file *pfile, #if defined(MODS_TEGRA) && defined(CONFIG_OF_IRQ) && defined(CONFIG_OF) int esc_mods_map_irq(struct file *pfile, - struct MODS_DT_INFO *p) + struct MODS_DT_INFO *p) { - int ret; + int err; /* the physical irq */ int hwirq; /* platform device handle */ @@ -1474,10 +1480,10 @@ int esc_mods_map_irq(struct file *pfile, np = of_find_node_by_name(np, p->dt_name); p->irq = irq_of_parse_and_map(np, p->index); - ret = of_irq_parse_one(np, p->index, &oirq); - if (ret) { + err = of_irq_parse_one(np, p->index, &oirq); + if (err) { mods_error_printk("Could not parse IRQ\n"); - return -EINVAL; + return err; } hwirq = oirq.args[1]; @@ -1499,7 +1505,6 @@ int esc_mods_map_irq(struct file *pfile, } /* enable the interrupt */ - return 0; - + return OK; } #endif diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c index 15e13ac2..949695f8 100644 --- a/drivers/misc/mods/mods_krnl.c +++ b/drivers/misc/mods/mods_krnl.c @@ -103,7 +103,7 @@ static const struct pci_device_id mods_pci_table[] = { .class = (PCI_CLASS_DISPLAY_3D << 8), .class_mask = ~0 }, - { } + { 0 } }; static int mods_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) @@ -137,8 +137,6 @@ static u32 access_token = MODS_ACCESS_TOKEN_NONE; #ifdef MODS_HAS_SRIOV static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs) { - int rv = 0; - LOG_ENT(); mods_debug_printk(DEBUG_PCI, @@ -148,57 +146,53 @@ static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs) dev->is_physfn); if (numvfs > 0) { - rv = pci_enable_sriov(dev, numvfs); - if (rv) { - mods_error_printk( - "pci_enable_sriov failed error %d\n", rv); - return rv; - } + int err = pci_enable_sriov(dev, numvfs); - rv = numvfs; + if (err) { + mods_error_printk("pci_enable_sriov failed with %d\n", + err); + numvfs = 0; + } } else { pci_disable_sriov(dev); - rv = 0; + numvfs = 0; } LOG_EXT(); - return rv; + return numvfs; } static int esc_mods_set_num_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) { - int rv = 0; - struct pci_dev *dev; - unsigned int devfn; + int err; + struct pci_dev *dev = NULL; LOG_ENT(); /* Get the PCI device structure for the specified device from kernel */ - devfn = PCI_DEVFN(p->dev.device, p->dev.function); - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { - mods_error_printk( - "unknown dev %04x:%02x:%02x.%x\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); + err = mods_find_pci_dev(pfile, &p->dev, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->dev.domain, + p->dev.bus, + p->dev.device, + p->dev.function); LOG_EXT(); - return -EINVAL; + return err; } - rv = mods_pci_sriov_configure(dev, p->numvfs); + err = mods_pci_sriov_configure(dev, p->numvfs); + pci_dev_put(dev); LOG_EXT(); - - return rv; + return err; } static int esc_mods_set_total_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) { - int rv = 0; - struct pci_dev *dev; - unsigned int devfn; + int err; + struct pci_dev *dev = NULL; LOG_ENT(); @@ -206,30 +200,26 @@ static int esc_mods_set_total_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) "pci_sriov_set_totalvfs(totalvfs=%d)\n", p->numvfs); /* Get the PCI device structure for the specified device from kernel */ - devfn = PCI_DEVFN(p->dev.device, p->dev.function); - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { - mods_error_printk( - "unknown dev %04x:%02x:%02x.%x\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); + err = mods_find_pci_dev(pfile, &p->dev, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->dev.domain, + p->dev.bus, + p->dev.device, + p->dev.function); LOG_EXT(); return -EINVAL; } - rv = pci_sriov_set_totalvfs(dev, p->numvfs); - - if (rv) { - mods_error_printk( - "pci_sriov_set_totalvfs failed error %d\n", rv); - return rv; - } + err = pci_sriov_set_totalvfs(dev, p->numvfs); + if (err) + mods_error_printk("pci_sriov_set_totalvfs failed with %d\n", + err); + pci_dev_put(dev); LOG_EXT(); - - return rv; + return err; } #endif @@ -412,7 +402,7 @@ static void mods_disable_all_devices(struct mods_client *client) if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex()))) return; - while (client->enabled_devices != 0) { + while (client->enabled_devices != NULL) { struct en_dev_entry *old = client->enabled_devices; mods_disable_device(old->dev); @@ -421,8 +411,13 @@ static void mods_disable_all_devices(struct mods_client *client) } mutex_unlock(mods_get_irq_mutex()); + + if (client->cached_dev) { + pci_dev_put(client->cached_dev); + client->cached_dev = NULL; + } #else - WARN_ON(client->enabled_devices != 0); + WARN_ON(client->enabled_devices != NULL); #endif } @@ -690,7 +685,7 @@ static int mods_krnl_close(struct inode *ip, struct file *fp) { struct mods_client *client = fp->private_data; u8 client_id = client->client_id; - int ret = OK; + int err = OK; LOG_ENT(); @@ -705,17 +700,17 @@ static int mods_krnl_close(struct inode *ip, struct file *fp) mods_resume_console(fp); mods_unregister_all_mappings(fp); - ret = mods_unregister_all_alloc(fp); - if (ret) + err = mods_unregister_all_alloc(fp); + if (err) mods_error_printk("failed to free all memory\n"); #if defined(CONFIG_PPC64) - ret = mods_unregister_all_ppc_tce_bypass(fp); - if (ret) + err = mods_unregister_all_ppc_tce_bypass(fp); + if (err) mods_error_printk("failed to restore dma bypass\n"); - ret = mods_unregister_all_nvlink_sysmem_trained(fp); - if (ret) + err = mods_unregister_all_nvlink_sysmem_trained(fp); + if (err) mods_error_printk("failed to free nvlink trained\n"); #endif @@ -725,7 +720,7 @@ static int mods_krnl_close(struct inode *ip, struct file *fp) mods_info_printk("driver closed\n"); LOG_EXT(); - return ret; + return err; } static unsigned int mods_krnl_poll(struct file *fp, poll_table *wait) @@ -781,17 +776,17 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma) mods_krnl_vma_open(vma); { - int ret = OK; + int err = OK; struct mods_client *client = fp->private_data; if (unlikely(mutex_lock_interruptible(&client->mtx))) - ret = -EINTR; + err = -EINTR; else { - ret = mods_krnl_map_inner(fp, vma); + err = mods_krnl_map_inner(fp, vma); mutex_unlock(&client->mtx); } LOG_EXT(); - return ret; + return err; } } @@ -810,41 +805,37 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) /* system memory */ if (p_mem_info) { u32 first, i; - struct MODS_PHYS_CHUNK *pt = p_mem_info->pages; + struct MODS_PHYS_CHUNK *chunks = p_mem_info->pages; u32 have_pages = 0; unsigned long map_va = 0; const pgprot_t prot = mods_get_prot(p_mem_info->cache_type, vma->vm_page_prot); /* Find the beginning of the requested range */ - for (first = 0; first < p_mem_info->max_chunks; first++) { - u64 dma_addr; + for (first = 0; first < p_mem_info->num_chunks; first++) { + u64 dma_addr = chunks[first].dma_addr; + u32 size = PAGE_SIZE << chunks[first].order; - if (!pt[first].allocated) - continue; - dma_addr = pt[first].dma_addr; if ((req_pa >= dma_addr) && - (req_pa < dma_addr + (PAGE_SIZE << pt->order))) { + (req_pa < dma_addr + size)) { break; } } - if (first == p_mem_info->max_chunks) { + if (first == p_mem_info->num_chunks) { mods_error_printk("can't satisfy requested mapping\n"); return -EINVAL; } /* Count how many remaining pages we have in the allocation */ - for (i = first; i < p_mem_info->max_chunks; i++) { - if (!pt[i].allocated) - break; + for (i = first; i < p_mem_info->num_chunks; i++) { if (i == first) { - u64 aoffs = req_pa - pt[i].dma_addr; + u64 aoffs = req_pa - chunks[i].dma_addr; u32 skip_pages = aoffs >> PAGE_SHIFT; have_pages -= skip_pages; } - have_pages += 1U << pt[i].order; + have_pages += 1U << chunks[i].order; } if (have_pages < req_pages) { @@ -856,15 +847,12 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) map_va = vma->vm_start; have_pages = req_pages; for (i = first; have_pages > 0; i++) { - u64 map_pa = MODS_DMA_TO_PHYS(pt[i].dma_addr); - u32 map_size = PAGE_SIZE << pt[i].order; - u32 map_pages = 1U << pt[i].order; - - if (!pt[i].allocated) - break; + u64 map_pa = MODS_DMA_TO_PHYS(chunks[i].dma_addr); + u32 map_size = PAGE_SIZE << chunks[i].order; + u32 map_pages = 1U << chunks[i].order; if (i == first) { - u64 aoffs = req_pa - pt[i].dma_addr; + u64 aoffs = req_pa - chunks[i].dma_addr; map_pa += aoffs; map_size -= aoffs; @@ -899,7 +887,7 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) */ mods_register_mapping(fp, p_mem_info, - pt[first].dma_addr, + chunks[first].dma_addr, vma->vm_start, MODS_VMA_SIZE(vma)); @@ -1034,7 +1022,7 @@ static int esc_mods_unlock_console(struct file *pfile) static int esc_mods_suspend_console(struct file *pfile) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); @@ -1052,7 +1040,7 @@ static int esc_mods_suspend_console(struct file *pfile) } console_unlock(); } - ret = OK; + err = OK; } #endif @@ -1064,13 +1052,13 @@ static int esc_mods_suspend_console(struct file *pfile) console_lock(); do_take_over_console(&dummy_con, 0, 0, 0); console_unlock(); - ret = OK; + err = OK; } #endif LOG_EXT(); - return ret; + return err; } static int esc_mods_resume_console(struct file *pfile) @@ -1080,7 +1068,7 @@ static int esc_mods_resume_console(struct file *pfile) static int mods_resume_console(struct file *pfile) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); @@ -1097,7 +1085,7 @@ static int mods_resume_console(struct file *pfile) } console_unlock(); } - ret = OK; + err = OK; } #endif @@ -1109,19 +1097,19 @@ static int mods_resume_console(struct file *pfile) console_lock(); do_unbind_con_driver(vc_cons[fg_console].d->vc_sw, 0, 0, 0); console_unlock(); - ret = OK; + err = OK; } #endif LOG_EXT(); - return ret; + return err; } static int esc_mods_acquire_access_token(struct file *pfile, struct MODS_ACCESS_TOKEN *ptoken) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); @@ -1129,14 +1117,14 @@ static int esc_mods_acquire_access_token(struct file *pfile, LOG_EXT(); mods_error_printk( "access token ops not supported with multi_instance=1!\n"); - return ret; + return err; } get_random_bytes(&ptoken->token, sizeof(ptoken->token)); - ret = mods_set_access_token(ptoken->token); - if (ret < 0) { + err = mods_set_access_token(ptoken->token); + if (err) mods_error_printk("unable to set access token!\n"); - } else { + else { struct mods_client *client = pfile->private_data; client->access_token = ptoken->token; @@ -1144,13 +1132,13 @@ static int esc_mods_acquire_access_token(struct file *pfile, LOG_EXT(); - return ret; + return err; } static int esc_mods_release_access_token(struct file *pfile, struct MODS_ACCESS_TOKEN *ptoken) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); @@ -1158,13 +1146,13 @@ static int esc_mods_release_access_token(struct file *pfile, LOG_EXT(); mods_error_printk( "access token ops not supported with multi_instance=1!\n"); - return ret; + return err; } - ret = mods_set_access_token(MODS_ACCESS_TOKEN_NONE); - if (ret < 0) { + err = mods_set_access_token(MODS_ACCESS_TOKEN_NONE); + if (err) mods_error_printk("unable to clear access token!\n"); - } else { + else { struct mods_client *client = pfile->private_data; client->access_token = MODS_ACCESS_TOKEN_NONE; @@ -1172,13 +1160,13 @@ static int esc_mods_release_access_token(struct file *pfile, LOG_EXT(); - return ret; + return err; } static int esc_mods_verify_access_token(struct file *pfile, struct MODS_ACCESS_TOKEN *ptoken) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); @@ -1186,19 +1174,19 @@ static int esc_mods_verify_access_token(struct file *pfile, struct mods_client *client = pfile->private_data; client->access_token = ptoken->token; - ret = OK; + err = OK; } else mods_error_printk("invalid access token\n"); LOG_EXT(); - return ret; + return err; } struct mods_sysfs_work { struct work_struct work; struct MODS_SYSFS_NODE *pdata; - int ret; + int err; }; #ifdef MODS_OLD_INIT_WORK @@ -1215,18 +1203,18 @@ static void sysfs_write_task(struct work_struct *w) LOG_ENT(); - task->ret = -EINVAL; + task->err = -EINVAL; old_fs = get_fs(); set_fs(KERNEL_DS); f = filp_open(task->pdata->path, O_WRONLY, 0); if (IS_ERR(f)) - task->ret = PTR_ERR(f); + task->err = PTR_ERR(f); else { f->f_pos = 0; if (task->pdata->size <= MODS_MAX_SYSFS_FILE_SIZE) - task->ret = f->f_op->write(f, + task->err = f->f_op->write(f, task->pdata->contents, task->pdata->size, &f->f_pos); @@ -1241,7 +1229,7 @@ static void sysfs_write_task(struct work_struct *w) static int esc_mods_write_sysfs_node(struct file *pfile, struct MODS_SYSFS_NODE *pdata) { - int ret = -EINVAL; + int err = -EINVAL; struct mods_sysfs_work task; struct workqueue_struct *wq; @@ -1256,7 +1244,7 @@ static int esc_mods_write_sysfs_node(struct file *pfile, wq = create_singlethread_workqueue("mods_sysfs_write"); if (!wq) { LOG_EXT(); - return ret; + return err; } #ifdef MODS_OLD_INIT_WORK @@ -1268,41 +1256,41 @@ static int esc_mods_write_sysfs_node(struct file *pfile, flush_workqueue(wq); destroy_workqueue(wq); - ret = task.ret; - if (ret > 0) - ret = OK; + err = task.err; + if (err > 0) + err = OK; LOG_EXT(); - return ret; + return err; } #ifdef MODS_HAS_MSR static int esc_mods_read_msr(struct file *pfile, struct MODS_MSR *p) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); - ret = rdmsr_safe_on_cpu(p->cpu_num, p->reg, &p->low, &p->high); - if (ret < 0) + err = rdmsr_safe_on_cpu(p->cpu_num, p->reg, &p->low, &p->high); + if (err) mods_error_printk("Could not read MSR %u\n", p->reg); LOG_EXT(); - return ret; + return err; } static int esc_mods_write_msr(struct file *pfile, struct MODS_MSR *p) { - int ret = -EINVAL; + int err = -EINVAL; LOG_ENT(); - ret = wrmsr_safe_on_cpu(p->cpu_num, p->reg, p->low, p->high); - if (ret < 0) + err = wrmsr_safe_on_cpu(p->cpu_num, p->reg, p->low, p->high); + if (err) mods_error_printk("Could not write MSR %u\n", p->reg); LOG_EXT(); - return ret; + return err; } #endif @@ -1314,7 +1302,7 @@ static long mods_krnl_ioctl(struct file *fp, unsigned int cmd, unsigned long i_arg) { - int ret = 0; + int err = 0; void *arg_copy = 0; void *arg = (void *) i_arg; int arg_size; @@ -1324,10 +1312,10 @@ static long mods_krnl_ioctl(struct file *fp, if ((cmd != MODS_ESC_VERIFY_ACCESS_TOKEN) && (cmd != MODS_ESC_GET_API_VERSION)) { - ret = mods_check_access_token(fp); - if (ret < 0) { + err = mods_check_access_token(fp); + if (err) { LOG_EXT(); - return ret; + return err; } } @@ -1354,15 +1342,15 @@ static long mods_krnl_ioctl(struct file *fp, do {\ mods_debug_printk(DEBUG_IOCTL, "ioctl(" #code ")\n");\ if (arg_size != sizeof(struct argtype)) {\ - ret = -EINVAL;\ + err = -EINVAL;\ mods_error_printk( \ "invalid parameter passed to ioctl " #code \ "\n");\ } else {\ - ret = function(fp, (struct argtype *)arg_copy);\ - if ((ret == OK) && \ + err = function(fp, (struct argtype *)arg_copy);\ + if ((err == OK) && \ copy_to_user(arg, arg_copy, arg_size)) {\ - ret = -EFAULT;\ + err = -EFAULT;\ mods_error_printk( \ "copying return value for ioctl " \ #code " to user space failed\n");\ @@ -1376,12 +1364,12 @@ static long mods_krnl_ioctl(struct file *fp, do {\ mods_debug_printk(DEBUG_IOCTL, "ioctl(" #code ")\n");\ if (arg_size != sizeof(struct argtype)) {\ - ret = -EINVAL;\ + err = -EINVAL;\ mods_error_printk( \ "invalid parameter passed to ioctl " #code \ "\n");\ } else {\ - ret = function(fp, (struct argtype *)arg_copy);\ + err = function(fp, (struct argtype *)arg_copy);\ } \ } while (0);\ }) @@ -1391,12 +1379,12 @@ static long mods_krnl_ioctl(struct file *fp, do {\ mods_debug_printk(DEBUG_IOCTL, "ioctl(" #code ")\n");\ if (arg_size != 0) {\ - ret = -EINVAL;\ + err = -EINVAL;\ mods_error_printk( \ "invalid parameter passed to ioctl " #code \ "\n");\ } else {\ - ret = function(fp);\ + err = function(fp);\ } \ } while (0);\ }) @@ -1558,6 +1546,12 @@ static long mods_krnl_ioctl(struct file *fp, MODS_GET_PHYSICAL_ADDRESS_3); break; + case MODS_ESC_GET_PHYSICAL_ADDRESS_RANGE: + MODS_IOCTL(MODS_ESC_GET_PHYSICAL_ADDRESS_RANGE, + esc_mods_get_phys_addr_range, + MODS_GET_ADDRESS_RANGE); + break; + case MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS: MODS_IOCTL(MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS, esc_mods_get_mapped_phys_addr, @@ -1576,6 +1570,12 @@ static long mods_krnl_ioctl(struct file *fp, MODS_GET_PHYSICAL_ADDRESS_3); break; + case MODS_ESC_GET_DMA_ADDRESS_RANGE: + MODS_IOCTL(MODS_ESC_GET_DMA_ADDRESS_RANGE, + esc_mods_get_dma_addr_range, + MODS_GET_ADDRESS_RANGE); + break; + case MODS_ESC_SET_MEMORY_TYPE: MODS_IOCTL_NORETVAL(MODS_ESC_SET_MEMORY_TYPE, esc_mods_set_mem_type, @@ -1617,6 +1617,7 @@ static long mods_krnl_ioctl(struct file *fp, break; #endif +#ifdef CONFIG_PCI case MODS_ESC_DMA_MAP_MEMORY: MODS_IOCTL(MODS_ESC_DMA_MAP_MEMORY, esc_mods_dma_map_memory, @@ -1628,10 +1629,11 @@ static long mods_krnl_ioctl(struct file *fp, esc_mods_dma_unmap_memory, MODS_DMA_MAP_MEMORY); break; +#endif case MODS_ESC_IRQ_REGISTER: case MODS_ESC_MSI_REGISTER: - ret = -EINVAL; + err = -EINVAL; break; case MODS_ESC_MAP_INTERRUPT: @@ -1727,7 +1729,7 @@ static long mods_krnl_ioctl(struct file *fp, /* fallthrough */ case MODS_ESC_ACPI_GET_DDC_2: /* Silent failure to avoid clogging kernel log */ - ret = -EINVAL; + err = -EINVAL; break; #endif case MODS_ESC_GET_API_VERSION: @@ -2036,7 +2038,7 @@ static long mods_krnl_ioctl(struct file *fp, mods_error_printk("unrecognized ioctl (0x%x) dir(0x%x) type (0x%x) nr (0x%x) size (0x%x)\n", cmd, _IOC_DIR(cmd), _IOC_TYPE(cmd), _IOC_NR(cmd), _IOC_SIZE(cmd)); - ret = -EINVAL; + err = -EINVAL; break; } @@ -2044,5 +2046,5 @@ static long mods_krnl_ioctl(struct file *fp, kfree(arg_copy); LOG_EXT(); - return ret; + return err; } diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c index 2e5bbd9a..69cc39bb 100644 --- a/drivers/misc/mods/mods_mem.c +++ b/drivers/misc/mods/mods_mem.c @@ -31,140 +31,199 @@ #include #endif -static int mods_post_alloc(struct MODS_PHYS_CHUNK *pt, +static int mods_post_alloc(struct MODS_PHYS_CHUNK *chunk, u64 phys_addr, struct MODS_MEM_INFO *p_mem_info); -static void mods_pre_free(struct MODS_PHYS_CHUNK *pt, +static void mods_pre_free(struct MODS_PHYS_CHUNK *chunk, struct MODS_MEM_INFO *p_mem_info); -static u64 mods_compress_nvlink_addr(struct pci_dev *dev, u64 addr); -static u64 mods_expand_nvlink_addr(struct pci_dev *dev, u64 addr47); - /**************************** * DMA MAP HELPER FUNCTIONS * ****************************/ -/* Unmap a page if it was mapped */ -static void mods_dma_unmap_page(struct MODS_DMA_MAP *p_dma_map, - struct MODS_MAP_CHUNK *pm) +/* + * Starting on Power9 systems, DMA addresses for NVLink are no longer + * the same as used over PCIE. + * + * Power9 supports a 56-bit Real Address. This address range is compressed + * when accessed over NvLink to allow the GPU to access all of memory using + * its 47-bit Physical address. + * + * If there is an NPU device present on the system, it implies that NvLink + * sysmem links are present and we need to apply the required address + * conversion for NvLink within the driver. This is intended to be temporary + * to ease the transition to kernel APIs to handle NvLink DMA mappings + * via the NPU device. + * + * Note, a deviation from the documented compression scheme is that the + * upper address bits (i.e. bit 56-63) instead of being set to zero are + * preserved during NvLink address compression so the orignal PCIE DMA + * address can be reconstructed on expansion. These bits can be safely + * ignored on NvLink since they are truncated by the GPU. + */ +#if defined(CONFIG_PPC64) && defined(CONFIG_PCI) +static u64 mods_compress_nvlink_addr(struct pci_dev *dev, u64 addr) { - if (!pm->pt) - return; + u64 addr47 = addr; - pm->map_addr = mods_expand_nvlink_addr(p_dma_map->dev, pm->map_addr); + /* Note, one key difference from the documented compression scheme + * is that BIT59 used for TCE bypass mode on PCIe is preserved during + * NVLink address compression to allow for the resulting DMA address to + * be used transparently on PCIe. + */ + if (has_npu_dev(dev, 0)) { + addr47 = addr & (1LLU << 59); + addr47 |= ((addr >> 45) & 0x3) << 43; + addr47 |= ((addr >> 49) & 0x3) << 45; + addr47 |= addr & ((1LLU << 43) - 1); + } - pci_unmap_page(p_dma_map->dev, - pm->map_addr, - (1U<pt->order)*PAGE_SIZE, + return addr47; +} +#else +#define mods_compress_nvlink_addr(dev, addr) (addr) +#endif + +#if defined(CONFIG_PPC64) && defined(CONFIG_PCI) +static u64 mods_expand_nvlink_addr(struct pci_dev *dev, u64 addr47) +{ + u64 addr = addr47; + + if (has_npu_dev(dev, 0)) { + addr = addr47 & ((1LLU << 43) - 1); + addr |= (addr47 & (3ULL << 43)) << 2; + addr |= (addr47 & (3ULL << 45)) << 4; + addr |= addr47 & ~((1ULL << 56) - 1); + } + + return addr; +} +#else +#define mods_expand_nvlink_addr(dev, addr) (addr) +#endif + +#ifdef CONFIG_PCI +/* Unmap a page if it was mapped */ +static void mods_dma_unmap_page(struct pci_dev *dev, + u64 dev_addr, + u32 order) +{ + dev_addr = mods_expand_nvlink_addr(dev, dev_addr); + + pci_unmap_page(dev, + dev_addr, + PAGE_SIZE << order, DMA_BIDIRECTIONAL); mods_debug_printk(DEBUG_MEM_DETAILED, - "dma unmap map_addr=0x%llx dma_addr=0x%llx on dev %04x:%02x:%02x.%x\n", - (unsigned long long)pm->map_addr, - (unsigned long long)pm->pt->dma_addr, - pci_domain_nr(p_dma_map->dev->bus), - p_dma_map->dev->bus->number, - PCI_SLOT(p_dma_map->dev->devfn), - PCI_FUNC(p_dma_map->dev->devfn)); + "dma unmap dev_addr=0x%llx on dev %04x:%02x:%02x.%x\n", + (unsigned long long)dev_addr, + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); } /* Unmap and delete the specified DMA mapping */ static int mods_dma_unmap_and_free(struct MODS_MEM_INFO *p_mem_info, - struct MODS_DMA_MAP *p_del_map) + struct MODS_DMA_MAP *p_del_map) { - struct MODS_DMA_MAP *p_dma_map; - - struct list_head *head; - struct list_head *iter; - - head = &p_mem_info->dma_map_list; + int found = 0; + struct list_head *head = &p_mem_info->dma_map_list; + struct list_head *iter; list_for_each(iter, head) { - p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list); + struct MODS_DMA_MAP *p_dma_map = list_entry(iter, + struct MODS_DMA_MAP, + list); - /* find the mapping to delete and remove it from the list */ - if (p_del_map == p_dma_map) { + if (p_dma_map == p_del_map) { list_del(iter); - - /* Safeguard check, all mappings should have a * - * non-null device - */ - if (p_dma_map->dev != NULL) { - int i; - - for (i = p_mem_info->max_chunks; i > 0; ) { - struct MODS_MAP_CHUNK *pm; - --i; - pm = &p_dma_map->mapping[i]; - mods_dma_unmap_page(p_dma_map, pm); - } - } - - kfree(p_dma_map); - - return OK; + found = 1; + break; } } - mods_error_printk("failed to unmap and free %p\n", - p_del_map); - return -EINVAL; + if (!found) { + mods_error_printk("failed to unmap and free %p\n", + p_del_map); + return -EINVAL; + } + + /* Safeguard check, all mappings should have a + * non-null device + */ + if (p_del_map->dev) { + int i; + + for (i = 0; i < p_mem_info->num_chunks; i++) + mods_dma_unmap_page(p_del_map->dev, + p_del_map->dev_addr[i], + p_mem_info->pages[i].order); + pci_dev_put(p_del_map->dev); + } + + kfree(p_del_map); + + return OK; } +#endif /* Unmap and delete all DMA mappings on the specified allocation */ int mods_dma_unmap_all(struct MODS_MEM_INFO *p_mem_info, - struct pci_dev *p_pci_dev) + struct pci_dev *dev) { +#ifdef CONFIG_PCI + int err = OK; struct list_head *head = &p_mem_info->dma_map_list; struct list_head *iter; struct list_head *tmp; list_for_each_safe(iter, tmp, head) { struct MODS_DMA_MAP *p_dma_map; - int ret; p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list); - if (!p_pci_dev || (p_dma_map->dev == p_pci_dev)) { - ret = mods_dma_unmap_and_free(p_mem_info, p_dma_map); - if (ret || p_pci_dev) - return ret; + if (!dev || (p_dma_map->dev == dev)) { + err = mods_dma_unmap_and_free(p_mem_info, p_dma_map); + if (err || dev) + break; } } + return err; +#else return OK; +#endif } +#ifdef CONFIG_PCI /* DMA map all pages in an allocation */ static void mods_dma_map_pages(struct MODS_MEM_INFO *p_mem_info, - struct MODS_DMA_MAP *p_dma_map) + struct MODS_DMA_MAP *p_dma_map) { int i; + struct pci_dev *dev = p_dma_map->dev; - for (i = p_mem_info->max_chunks; i > 0; ) { - struct MODS_MAP_CHUNK *pm; - struct MODS_PHYS_CHUNK *pt; - --i; - pm = &p_dma_map->mapping[i]; - pt = &p_mem_info->pages[i]; - if (!pt->allocated) - continue; + for (i = 0; i < p_mem_info->num_chunks; i++) { + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[i]; + u64 dev_addr; - pm->pt = pt; - pm->map_addr = pci_map_page(p_dma_map->dev, - pt->p_page, - 0, - (1U << pt->order) * PAGE_SIZE, - DMA_BIDIRECTIONAL); + dev_addr = pci_map_page(dev, + chunk->p_page, + 0, + PAGE_SIZE << chunk->order, + DMA_BIDIRECTIONAL); - pm->map_addr = mods_compress_nvlink_addr(p_dma_map->dev, - pm->map_addr); + dev_addr = mods_compress_nvlink_addr(dev, dev_addr); + + p_dma_map->dev_addr[i] = dev_addr; mods_debug_printk(DEBUG_MEM_DETAILED, - "dma map map_addr=0x%llx, dma_addr=0x%llx on dev %04x:%02x:%02x.%x\n", - (unsigned long long)pm->map_addr, - (unsigned long long)pt->dma_addr, + "dma map dev_addr=0x%llx, phys_addr=0x%llx on dev %04x:%02x:%02x.%x\n", + (unsigned long long)dev_addr, + (unsigned long long)chunk->dma_addr, pci_domain_nr(p_dma_map->dev->bus), p_dma_map->dev->bus->number, PCI_SLOT(p_dma_map->dev->devfn), @@ -172,18 +231,17 @@ static void mods_dma_map_pages(struct MODS_MEM_INFO *p_mem_info, } } -/* Create a DMA map on the specified allocation for the pci device. Lazy * - * initialize the map list structure if one does not yet exist. +/* Create a DMA map on the specified allocation for the pci device. + * Lazy-initialize the map list structure if one does not yet exist. */ static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, - struct pci_dev *p_pci_dev) + struct pci_dev *dev) { struct MODS_DMA_MAP *p_dma_map; u32 alloc_size; alloc_size = sizeof(*p_dma_map) + - (p_mem_info->max_chunks - 1) * - sizeof(struct MODS_MAP_CHUNK); + (p_mem_info->num_chunks - 1) * sizeof(u64); p_dma_map = kzalloc(alloc_size, GFP_KERNEL | __GFP_NORETRY); if (unlikely(!p_dma_map)) { @@ -191,45 +249,30 @@ static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, return -ENOMEM; } - p_dma_map->dev = p_pci_dev; + p_dma_map->dev = pci_dev_get(dev); mods_dma_map_pages(p_mem_info, p_dma_map); list_add(&p_dma_map->list, &p_mem_info->dma_map_list); return OK; } +#endif -/* Find the dma mapping chunk for the specified memory. If p_phys_chunk is * - * NULL then the first mapped chunk is returned. - */ -static struct MODS_MAP_CHUNK *mods_find_dma_map_chunk( - struct MODS_MEM_INFO *p_mem_info, - struct pci_dev *p_pci_dev, - struct MODS_PHYS_CHUNK *p_phys_chunk) +/* Find the dma mapping chunk for the specified memory. */ +static struct MODS_DMA_MAP *find_dma_map(struct MODS_MEM_INFO *p_mem_info, + struct mods_pci_dev_2 *pcidev) { - struct MODS_DMA_MAP *p_dma_map; - struct list_head *head; - struct list_head *iter; - int i; + struct MODS_DMA_MAP *p_dma_map = NULL; + struct list_head *head = &p_mem_info->dma_map_list; + struct list_head *iter; - head = &p_mem_info->dma_map_list; if (!head) return NULL; list_for_each(iter, head) { p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list); - if (p_dma_map->dev == p_pci_dev) { - if (!p_phys_chunk) - return &p_dma_map->mapping[0]; - - for (i = p_mem_info->max_chunks; i > 0;) { - struct MODS_MAP_CHUNK *pm; - --i; - pm = &p_dma_map->mapping[i]; - if (pm->pt == p_phys_chunk) - return pm; - } - } + if (mods_is_pci_dev(p_dma_map->dev, pcidev)) + return p_dma_map; } return NULL; } @@ -260,13 +303,13 @@ static void mods_restore_cache(struct MODS_MEM_INFO *p_mem_info) { unsigned int i; - for (i = 0; i < p_mem_info->max_chunks; i++) { - struct MODS_PHYS_CHUNK *pt = &p_mem_info->pages[i]; + for (i = 0; i < p_mem_info->num_chunks; i++) { + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[i]; - if (!pt->allocated) + if (!chunk->allocated) break; - mods_pre_free(pt, p_mem_info); + mods_pre_free(chunk, p_mem_info); } } @@ -275,20 +318,35 @@ static void mods_free_pages(struct MODS_MEM_INFO *p_mem_info) unsigned int i; /* release in reverse order */ - for (i = p_mem_info->max_chunks; i > 0; ) { - struct MODS_PHYS_CHUNK *pt; + for (i = p_mem_info->num_chunks; i > 0; ) { + struct MODS_PHYS_CHUNK *chunk; + --i; - pt = &p_mem_info->pages[i]; - if (!pt->allocated) + chunk = &p_mem_info->pages[i]; + if (!chunk->allocated) continue; + if (p_mem_info->dev) { + u64 dev_addr = chunk->dev_addr; + + dev_addr = mods_expand_nvlink_addr(p_mem_info->dev, + dev_addr); + + pci_unmap_page(p_mem_info->dev, + dev_addr, + PAGE_SIZE << chunk->order, + DMA_BIDIRECTIONAL); + } + #ifdef CONFIG_BIGPHYS_AREA if (p_mem_info->alloc_type == MODS_ALLOC_TYPE_BIGPHYS_AREA) { bigphysarea_free_pages((void *) p_mem_info->logical_addr); } else #endif - __free_pages(pt->p_page, pt->order); + __free_pages(chunk->p_page, chunk->order); + + chunk->allocated = 0; } } @@ -394,9 +452,9 @@ static int mods_alloc_contig_sys_pages(struct MODS_MEM_INFO *p_mem_info) return 0; } -static int mods_get_max_order_needed(u32 num_pages) +static u32 mods_get_max_order_needed(u32 num_pages) { - int order = 0; + u32 order = 0; while (order < 10 && (1U<<(order+1)) <= num_pages) ++order; @@ -411,51 +469,54 @@ static int mods_alloc_noncontig_sys_pages(struct MODS_MEM_INFO *p_mem_info) LOG_ENT(); memset(p_mem_info->pages, 0, - p_mem_info->max_chunks * sizeof(p_mem_info->pages[0])); + p_mem_info->num_chunks * sizeof(p_mem_info->pages[0])); /* alloc pages */ while (pages_left > 0) { u64 phys_addr = 0; - int order = mods_get_max_order_needed(pages_left); - struct MODS_PHYS_CHUNK *pt = &p_mem_info->pages[num_chunks]; + u32 order = mods_get_max_order_needed(pages_left); + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[num_chunks]; - for ( ; order >= 0; --order) { - pt->p_page = alloc_pages_node( + for (;;) { + chunk->p_page = alloc_pages_node( p_mem_info->numa_node, mods_alloc_flags(p_mem_info), - (unsigned int)order); - if (pt->p_page) + order); + if (chunk->p_page) break; + if (order == 0) + break; + --order; } - if (!pt->p_page) { + if (!chunk->p_page) { mods_error_printk("out of memory\n"); goto failed; } - pt->allocated = 1; + chunk->allocated = 1; pages_left -= 1U << order; - pt->order = (u32)order; + chunk->order = order; - phys_addr = page_to_phys(pt->p_page); + phys_addr = page_to_phys(chunk->p_page); if (phys_addr == 0) { mods_error_printk("phys addr lookup failed\n"); goto failed; } - pt->dma_addr = MODS_PHYS_TO_DMA(phys_addr); + chunk->dma_addr = MODS_PHYS_TO_DMA(phys_addr); mods_debug_printk(DEBUG_MEM, "alloc 0x%lx bytes [%u], 2^%u pages, %s, node %d, addrbits %u, phys 0x%llx\n", (unsigned long)p_mem_info->length, (unsigned int)num_chunks, - pt->order, + chunk->order, mods_get_prot_str(p_mem_info->cache_type), p_mem_info->numa_node, (unsigned int)p_mem_info->addr_bits, - (unsigned long long)pt->dma_addr); + (unsigned long long)chunk->dma_addr); ++num_chunks; - if (mods_post_alloc(pt, phys_addr, p_mem_info)) + if (mods_post_alloc(chunk, phys_addr, p_mem_info)) goto failed; } @@ -505,6 +566,7 @@ static int mods_unregister_and_free(struct file *fp, mods_dma_unmap_all(p_mem_info, NULL); mods_restore_cache(p_mem_info); mods_free_pages(p_mem_info); + pci_dev_put(p_mem_info->dev); kfree(p_mem_info); @@ -521,6 +583,7 @@ static int mods_unregister_and_free(struct file *fp, int mods_unregister_all_alloc(struct file *fp) { + int err = OK; struct mods_client *client = fp->private_data; struct list_head *head = &client->mem_alloc_list; struct list_head *iter; @@ -528,15 +591,126 @@ int mods_unregister_all_alloc(struct file *fp) list_for_each_safe(iter, tmp, head) { struct MODS_MEM_INFO *p_mem_info; - int ret; p_mem_info = list_entry(iter, struct MODS_MEM_INFO, list); - ret = mods_unregister_and_free(fp, p_mem_info); - if (ret) - return ret; + err = mods_unregister_and_free(fp, p_mem_info); + if (err) + break; } - return OK; + return err; +} + +static int get_addr_range(struct file *fp, + struct MODS_GET_ADDRESS_RANGE *p, + struct mods_pci_dev_2 *pcidev) +{ + struct MODS_MEM_INFO *p_mem_info; + struct MODS_DMA_MAP *p_dma_map = NULL; + u64 *out; + u32 num_out; + u32 skip_pages; + u32 i; + int err = OK; + u32 page_offs; + + LOG_ENT(); + + p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; + if (unlikely(!p_mem_info)) { + mods_error_printk("no allocation given\n"); + LOG_EXT(); + return -EINVAL; + } + + if (unlikely(pcidev && (pcidev->bus > 0xFFU || + pcidev->device > 0xFFU))) { + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + pcidev->domain, + pcidev->bus, + pcidev->device, + pcidev->function); + LOG_EXT(); + return -EINVAL; + } + + if (unlikely(p->stride != PAGE_SIZE)) { + mods_error_printk("stride is 0x%x, but expected 0x%lx\n", + p->stride, PAGE_SIZE); + LOG_EXT(); + return -EINVAL; + } + + out = &p->physical_addresses[0]; + num_out = p->num_entries; + + if (unlikely(num_out == 0 || num_out > MAX_PA_ENTRIES)) { + mods_error_printk("invalid number of pages requested: %u\n", + num_out); + LOG_EXT(); + return -EINVAL; + } + + if (pcidev && !mods_is_pci_dev(p_mem_info->dev, pcidev)) { + p_dma_map = find_dma_map(p_mem_info, pcidev); + if (unlikely(!p_dma_map)) { + mods_error_printk("allocation %p is not mapped to device %04x:%02x:%02x.%x\n", + p_mem_info, + pcidev->domain, + pcidev->bus, + pcidev->device, + pcidev->function); + LOG_EXT(); + return -EINVAL; + } + } + + page_offs = p->offset & (~PAGE_MASK); + skip_pages = p->offset >> PAGE_SHIFT; + + for (i = 0; i < p_mem_info->num_chunks && num_out; i++) { + u32 num_pages; + u64 addr; + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[i]; + + num_pages = 1U << chunk->order; + if (num_pages <= skip_pages) { + skip_pages -= num_pages; + continue; + } + + addr = pcidev ? + (p_dma_map ? p_dma_map->dev_addr[i] : chunk->dev_addr) + : chunk->dma_addr; + + if (skip_pages) { + num_pages -= skip_pages; + addr += skip_pages << PAGE_SHIFT; + skip_pages = 0; + } + + if (num_pages > num_out) + num_pages = num_out; + + while (num_pages) { + *out = addr + page_offs; + ++out; + --num_out; + addr += PAGE_SIZE; + --num_pages; + } + } + + if (unlikely(num_out)) { + mods_error_printk("invalid offset 0x%llx or size 0x%llx requested for allocation %p\n", + p->offset, + (u64)p->stride * p->num_entries, + p_mem_info); + err = -EINVAL; + } + + LOG_EXT(); + return err; } /* Returns an offset within an allocation deduced from physical address. @@ -549,13 +723,10 @@ int mods_get_alloc_offset(struct MODS_MEM_INFO *p_mem_info, u32 i; u64 offset = 0; - for (i = 0; i < p_mem_info->max_chunks; i++) { - struct MODS_PHYS_CHUNK *pt = &p_mem_info->pages[i]; - u64 addr = pt->dma_addr; - u64 size = PAGE_SIZE << pt->order; - - if (!pt->allocated) - break; + for (i = 0; i < p_mem_info->num_chunks; i++) { + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[i]; + u64 addr = chunk->dma_addr; + u32 size = PAGE_SIZE << chunk->order; if (dma_addr >= addr && dma_addr < addr + size) { @@ -590,62 +761,64 @@ struct MODS_MEM_INFO *mods_find_alloc(struct file *fp, u64 phys_addr) return NULL; } -static u32 mods_estimate_max_chunks(u32 num_pages) +static u32 mods_estimate_num_chunks(u32 num_pages) { - u32 max_chunks = 0; + u32 num_chunks = 0; u32 bit_scan; /* Count each contiguous block <=256KB */ - for (bit_scan = num_pages; bit_scan && max_chunks < 6; bit_scan >>= 1) - ++max_chunks; + for (bit_scan = num_pages; bit_scan && num_chunks < 6; bit_scan >>= 1) + ++num_chunks; /* Count remaining contiguous blocks >256KB */ - max_chunks += bit_scan; + num_chunks += bit_scan; /* 4x slack for medium memory fragmentation */ - max_chunks <<= 2; + num_chunks <<= 2; /* No sense to allocate more chunks than pages */ - if (max_chunks > num_pages) - max_chunks = num_pages; + if (num_chunks > num_pages) + num_chunks = num_pages; /* Now, if memory is heavily fragmented, we are screwed */ - return max_chunks; + return num_chunks; } -static struct MODS_PHYS_CHUNK *mods_find_phys_chunk( - struct MODS_MEM_INFO *p_mem_info, - u64 offset, - u64 *chunk_offset) +/* For large non-contiguous allocations, we typically use significantly less + * chunks than originally estimated. This function reallocates the + * MODS_MEM_INFO struct so that it uses only as much memory as it needs. + */ +static struct MODS_MEM_INFO *optimize_chunks(struct MODS_MEM_INFO *p_mem_info) { - struct MODS_PHYS_CHUNK *pt = NULL; - u64 pages_left; - u64 page_offs; - u32 i; + u32 i; + u32 num_chunks; + u32 alloc_size = 0; + struct MODS_MEM_INFO *p_new_mem_info = NULL; - if (!p_mem_info) - return NULL; - - pages_left = offset >> PAGE_SHIFT; - page_offs = offset & (~PAGE_MASK); - - for (i = 0; i < p_mem_info->max_chunks; i++) { - u32 num_pages; - - pt = &p_mem_info->pages[i]; - if (!pt->allocated) + for (i = 0; i < p_mem_info->num_chunks; i++) + if (!p_mem_info->pages[i].allocated) break; - num_pages = 1U << pt->order; - if (pages_left < num_pages) - break; - pages_left -= num_pages; - pt = NULL; + num_chunks = i; + + if (num_chunks < p_mem_info->num_chunks) { + alloc_size = sizeof(*p_mem_info) + + (num_chunks - 1) * sizeof(struct MODS_PHYS_CHUNK); + + p_new_mem_info = kzalloc(alloc_size, + GFP_KERNEL | __GFP_NORETRY); } - *chunk_offset = (pages_left << PAGE_SHIFT) + page_offs; - return pt; + if (p_new_mem_info) { + memcpy(p_new_mem_info, p_mem_info, alloc_size); + p_new_mem_info->num_chunks = num_chunks; + INIT_LIST_HEAD(&p_new_mem_info->dma_map_list); + kfree(p_mem_info); + p_mem_info = p_new_mem_info; + } + + return p_mem_info; } /************************ @@ -658,24 +831,27 @@ int esc_mods_device_alloc_pages_2(struct file *fp, struct MODS_MEM_INFO *p_mem_info = NULL; u32 num_pages; u32 alloc_size; - u32 max_chunks; - int ret = OK; + u32 num_chunks; + int err = OK; struct pci_dev *dev = NULL; LOG_ENT(); if (!p->num_bytes) { mods_error_printk("zero bytes requested\n"); - ret = -EINVAL; + err = -EINVAL; goto failed; } - mods_debug_printk( - DEBUG_MEM_DETAILED, - "alloc 0x%x bytes %s %s\n", - p->num_bytes, - p->contiguous ? "contiguous" : "noncontiguous", - mods_get_prot_str(p->attrib)); + mods_debug_printk(DEBUG_MEM_DETAILED, + "alloc 0x%x bytes %s %s on %04x:%02x:%02x.%x\n", + p->num_bytes, + p->contiguous ? "contiguous" : "noncontiguous", + mods_get_prot_str(p->attrib), + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); switch (p->attrib) { case MODS_MEMORY_CACHED: @@ -688,27 +864,27 @@ int esc_mods_device_alloc_pages_2(struct file *fp, default: mods_error_printk("invalid memory type: %u\n", p->attrib); - ret = -ENOMEM; + err = -ENOMEM; goto failed; } num_pages = (u32)(((u64)p->num_bytes + PAGE_SIZE - 1) >> PAGE_SHIFT); if (p->contiguous) - max_chunks = 1; + num_chunks = 1; else - max_chunks = mods_estimate_max_chunks(num_pages); + num_chunks = mods_estimate_num_chunks(num_pages); alloc_size = sizeof(*p_mem_info) + - (max_chunks - 1) * sizeof(struct MODS_PHYS_CHUNK); + (num_chunks - 1) * sizeof(struct MODS_PHYS_CHUNK); p_mem_info = kzalloc(alloc_size, GFP_KERNEL | __GFP_NORETRY); if (unlikely(!p_mem_info)) { mods_error_printk("failed to allocate auxiliary 0x%x bytes\n", alloc_size); - ret = -ENOMEM; + err = -ENOMEM; goto failed; } - p_mem_info->max_chunks = max_chunks; + p_mem_info->num_chunks = num_chunks; p_mem_info->alloc_type = p->contiguous ? MODS_ALLOC_TYPE_CONTIG : MODS_ALLOC_TYPE_NON_CONTIG; p_mem_info->cache_type = p->attrib; @@ -721,43 +897,45 @@ int esc_mods_device_alloc_pages_2(struct file *fp, INIT_LIST_HEAD(&p_mem_info->dma_map_list); - if (p->pci_device.bus || p->pci_device.device) { - unsigned int devfn = PCI_DEVFN(p->pci_device.device, - p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, - p->pci_device.bus, - devfn); - if (!dev) { - ret = -EINVAL; + if (p->pci_device.bus <= 0xFFU && p->pci_device.device <= 0xFFU) { + err = mods_find_pci_dev(fp, &p->pci_device, &dev); + if (unlikely(err)) goto failed; - } p_mem_info->dev = dev; #if defined(MODS_HAS_DEV_TO_NUMA_NODE) p_mem_info->numa_node = dev_to_node(&dev->dev); #endif -#if defined(MODS_HAS_PNV_PCI_GET_NPU_DEV) - if (!mods_is_nvlink_sysmem_trained(fp, dev) && - pnv_pci_get_npu_dev(dev, 0)) - p_mem_info->numa_node = 0; +#if defined(CONFIG_PPC64) && defined(CONFIG_PCI) + if (!mods_is_nvlink_sysmem_trained(fp, dev)) { + /* Until NvLink is trained, we must use memory + * on node 0. + */ + if (has_npu_dev(dev, 0)) + p_mem_info->numa_node = 0; + } #endif mods_debug_printk(DEBUG_MEM_DETAILED, - "affinity %04x:%02x:%02x.%x node %d\n", - p->pci_device.domain, - p->pci_device.bus, - p->pci_device.device, - p->pci_device.function, - p_mem_info->numa_node); + "affinity %04x:%02x:%02x.%x node %d\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function, + p_mem_info->numa_node); } p->memory_handle = 0; if (p->contiguous) - ret = mods_alloc_contig_sys_pages(p_mem_info); - else - ret = mods_alloc_noncontig_sys_pages(p_mem_info); + err = mods_alloc_contig_sys_pages(p_mem_info); + else { + err = mods_alloc_noncontig_sys_pages(p_mem_info); - if (ret) { + if (!err) + p_mem_info = optimize_chunks(p_mem_info); + } + + if (err) { mods_error_printk( "failed to alloc 0x%x %s bytes, %s, node %d, addrbits %u\n", p_mem_info->length, @@ -765,39 +943,32 @@ int esc_mods_device_alloc_pages_2(struct file *fp, mods_get_prot_str(p_mem_info->cache_type), p_mem_info->numa_node, (unsigned int)p_mem_info->addr_bits); - ret = -ENOMEM; goto failed; } -#if defined(CONFIG_PPC64) - /* Backwards compatibility : this is normally done through - * MODS_ESC_DMA_MAP_MEMORY - */ - if (dev && mods_create_dma_map(p_mem_info, dev)) { - mods_error_printk("failed to create dma map\n"); - ret = -ENOMEM; - goto failed; - } -#endif - p->memory_handle = (u64)(size_t)p_mem_info; mods_debug_printk(DEBUG_MEM_DETAILED, "alloc %p\n", p_mem_info); - ret = mods_register_alloc(fp, p_mem_info); + err = mods_register_alloc(fp, p_mem_info); failed: - if (ret) + if (err) { + if (p_mem_info) { + mods_free_pages(p_mem_info); + pci_dev_put(p_mem_info->dev); + } kfree(p_mem_info); + } LOG_EXT(); - return ret; + return err; } int esc_mods_device_alloc_pages(struct file *fp, struct MODS_DEVICE_ALLOC_PAGES *p) { - int retval; + int err; struct MODS_DEVICE_ALLOC_PAGES_2 dev_alloc_pages = {0}; LOG_ENT(); @@ -811,17 +982,17 @@ int esc_mods_device_alloc_pages(struct file *fp, dev_alloc_pages.pci_device.device = p->pci_device.device; dev_alloc_pages.pci_device.function = p->pci_device.function; - retval = esc_mods_device_alloc_pages_2(fp, &dev_alloc_pages); - if (!retval) + err = esc_mods_device_alloc_pages_2(fp, &dev_alloc_pages); + if (!err) p->memory_handle = dev_alloc_pages.memory_handle; LOG_EXT(); - return retval; + return err; } int esc_mods_alloc_pages(struct file *fp, struct MODS_ALLOC_PAGES *p) { - int retval; + int err; struct MODS_DEVICE_ALLOC_PAGES_2 dev_alloc_pages; LOG_ENT(); @@ -830,31 +1001,31 @@ int esc_mods_alloc_pages(struct file *fp, struct MODS_ALLOC_PAGES *p) dev_alloc_pages.contiguous = p->contiguous; dev_alloc_pages.address_bits = p->address_bits; dev_alloc_pages.attrib = p->attrib; - dev_alloc_pages.pci_device.domain = 0; - dev_alloc_pages.pci_device.bus = 0; - dev_alloc_pages.pci_device.device = 0; - dev_alloc_pages.pci_device.function = 0; + dev_alloc_pages.pci_device.domain = 0xFFFFU; + dev_alloc_pages.pci_device.bus = 0xFFFFU; + dev_alloc_pages.pci_device.device = 0xFFFFU; + dev_alloc_pages.pci_device.function = 0xFFFFU; - retval = esc_mods_device_alloc_pages_2(fp, &dev_alloc_pages); - if (!retval) + err = esc_mods_device_alloc_pages_2(fp, &dev_alloc_pages); + if (!err) p->memory_handle = dev_alloc_pages.memory_handle; LOG_EXT(); - return retval; + return err; } int esc_mods_free_pages(struct file *fp, struct MODS_FREE_PAGES *p) { - int ret; + int err; LOG_ENT(); - ret = mods_unregister_and_free(fp, + err = mods_unregister_and_free(fp, (struct MODS_MEM_INFO *)(size_t)p->memory_handle); LOG_EXT(); - return ret; + return err; } int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) @@ -902,156 +1073,143 @@ int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) { - int retval; - struct MODS_GET_PHYSICAL_ADDRESS_3 get_phys_addr_3; + struct MODS_GET_ADDRESS_RANGE range; + int err; LOG_ENT(); - memset(&get_phys_addr_3, 0, - sizeof(struct MODS_GET_PHYSICAL_ADDRESS_3)); - get_phys_addr_3.memory_handle = p->memory_handle; - get_phys_addr_3.offset = p->offset; + range.memory_handle = p->memory_handle; + range.offset = p->offset; + range.stride = PAGE_SIZE; + range.num_entries = 1; + memset(&range.pci_device, 0, sizeof(range.pci_device)); - retval = esc_mods_get_phys_addr_2(fp, &get_phys_addr_3); - if (!retval) - p->physical_address = get_phys_addr_3.physical_address; + err = get_addr_range(fp, &range, NULL); + + if (!err) + p->physical_address = range.physical_addresses[0]; LOG_EXT(); - return retval; + return err; } int esc_mods_get_phys_addr_2(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_3 *p) { - struct MODS_MEM_INFO *p_mem_info; - struct MODS_PHYS_CHUNK *pt = NULL; - u64 chunk_offset; + struct MODS_GET_ADDRESS_RANGE range; + int err; LOG_ENT(); - p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; - pt = mods_find_phys_chunk(p_mem_info, p->offset, &chunk_offset); + range.memory_handle = p->memory_handle; + range.offset = p->offset; + range.stride = PAGE_SIZE; + range.num_entries = 1; + memset(&range.pci_device, 0, sizeof(range.pci_device)); - if (!pt || !pt->allocated) { - mods_error_printk("invalid offset 0x%llx requested for allocation %p\n", - p->offset, p_mem_info); - LOG_EXT(); - return -EINVAL; - } + err = get_addr_range(fp, &range, NULL); + + if (!err) + p->physical_address = range.physical_addresses[0]; - p->physical_address = pt->dma_addr + chunk_offset; - mods_debug_printk(DEBUG_MEM_DETAILED, - "get phys: %p+0x%llx -> 0x%llx\n", - p_mem_info, p->offset, p->physical_address); LOG_EXT(); - return 0; + return err; +} + +int esc_mods_get_phys_addr_range(struct file *fp, + struct MODS_GET_ADDRESS_RANGE *p) +{ + return get_addr_range(fp, p, NULL); } int esc_mods_get_mapped_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) { - int retval; - struct MODS_GET_PHYSICAL_ADDRESS_3 get_mapped_phys_addr_3; + struct MODS_GET_ADDRESS_RANGE range; struct MODS_MEM_INFO *p_mem_info; + int err; LOG_ENT(); - memset(&get_mapped_phys_addr_3, 0, - sizeof(struct MODS_GET_PHYSICAL_ADDRESS_3)); - get_mapped_phys_addr_3.memory_handle = p->memory_handle; - get_mapped_phys_addr_3.offset = p->offset; + range.memory_handle = p->memory_handle; + range.offset = p->offset; + range.stride = PAGE_SIZE; + range.num_entries = 1; + p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; if (p_mem_info->dev) { - get_mapped_phys_addr_3.pci_device.domain = + range.pci_device.domain = pci_domain_nr(p_mem_info->dev->bus); - get_mapped_phys_addr_3.pci_device.bus = + range.pci_device.bus = p_mem_info->dev->bus->number; - get_mapped_phys_addr_3.pci_device.device = + range.pci_device.device = PCI_SLOT(p_mem_info->dev->devfn); - get_mapped_phys_addr_3.pci_device.function = + range.pci_device.function = PCI_FUNC(p_mem_info->dev->devfn); + + err = get_addr_range(fp, &range, &range.pci_device); + } else { + memset(&range.pci_device, 0, sizeof(range.pci_device)); + err = get_addr_range(fp, &range, NULL); } - retval = esc_mods_get_mapped_phys_addr_3(fp, &get_mapped_phys_addr_3); - if (!retval) - p->physical_address = get_mapped_phys_addr_3.physical_address; + if (!err) + p->physical_address = range.physical_addresses[0]; LOG_EXT(); - return retval; + return err; } int esc_mods_get_mapped_phys_addr_2(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_2 *p) { - int retval; - struct MODS_GET_PHYSICAL_ADDRESS_3 get_mapped_phys_addr_3; + struct MODS_GET_ADDRESS_RANGE range; + int err; LOG_ENT(); - memset(&get_mapped_phys_addr_3, 0, - sizeof(struct MODS_GET_PHYSICAL_ADDRESS_3)); - get_mapped_phys_addr_3.memory_handle = p->memory_handle; - get_mapped_phys_addr_3.offset = p->offset; - get_mapped_phys_addr_3.pci_device = p->pci_device; + range.memory_handle = p->memory_handle; + range.offset = p->offset; + range.stride = PAGE_SIZE; + range.num_entries = 1; + range.pci_device = p->pci_device; - retval = esc_mods_get_mapped_phys_addr_3(fp, &get_mapped_phys_addr_3); - if (!retval) - p->physical_address = get_mapped_phys_addr_3.physical_address; + err = get_addr_range(fp, &range, &range.pci_device); + + if (!err) + p->physical_address = range.physical_addresses[0]; LOG_EXT(); - return retval; + return err; } int esc_mods_get_mapped_phys_addr_3(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_3 *p) { - struct pci_dev *dev = NULL; - struct MODS_MEM_INFO *p_mem_info; - struct MODS_PHYS_CHUNK *pt; - struct MODS_MAP_CHUNK *pm; - u64 chunk_offset; + struct MODS_GET_ADDRESS_RANGE range; + int err; - p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; - pt = mods_find_phys_chunk(p_mem_info, p->offset, &chunk_offset); + LOG_ENT(); - if (!pt || !pt->allocated) { - mods_error_printk("invalid offset 0x%llx requested for allocation %p\n", - p->offset, p_mem_info); - LOG_EXT(); - return -EINVAL; - } + range.memory_handle = p->memory_handle; + range.offset = p->offset; + range.stride = PAGE_SIZE; + range.num_entries = 1; + range.pci_device = p->pci_device; - if (p->pci_device.bus || p->pci_device.device) { - unsigned int devfn = PCI_DEVFN(p->pci_device.device, - p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, - p->pci_device.bus, - devfn); - } + err = get_addr_range(fp, &range, &range.pci_device); - if (!dev) { - mods_debug_printk(DEBUG_MEM_DETAILED, - "get mapped phys: %p+0x%llx -> 0x%llx\n", - p_mem_info, p->offset, p->physical_address); - p->physical_address = pt->dma_addr + chunk_offset; - LOG_EXT(); - return OK; - } + if (!err) + p->physical_address = range.physical_addresses[0]; - pm = mods_find_dma_map_chunk(p_mem_info, dev, pt); - if (!pm) { - mods_error_printk("invalid device mapping requested for allocation %p\n", - p_mem_info); - LOG_EXT(); - return -EINVAL; - } - - p->physical_address = pm->map_addr + chunk_offset; - mods_debug_printk(DEBUG_MEM_DETAILED, - "get mapped phys: %p+0x%llx -> 0x%llx\n", - p_mem_info, p->offset, p->physical_address); LOG_EXT(); - return 0; + return err; +} + +int esc_mods_get_dma_addr_range(struct file *fp, + struct MODS_GET_ADDRESS_RANGE *p) +{ + return get_addr_range(fp, p, &p->pci_device); } int esc_mods_virtual_to_phys(struct file *fp, @@ -1084,7 +1242,7 @@ int esc_mods_virtual_to_phys(struct file *fp, if (p->virtual_address >= begin && p->virtual_address < end) { u64 virt_offs = p->virtual_address - begin; - int ret; + int err; /* device memory mapping */ if (!p_map_mem->p_mem_info) { @@ -1112,9 +1270,9 @@ int esc_mods_virtual_to_phys(struct file *fp, mutex_unlock(&client->mtx); - ret = esc_mods_get_phys_addr(fp, &get_phys_addr); - if (ret != OK) - return ret; + err = esc_mods_get_phys_addr(fp, &get_phys_addr); + if (err) + return err; p->physical_address = get_phys_addr.physical_address; @@ -1221,82 +1379,100 @@ int esc_mods_memory_barrier(struct file *fp) #endif } +#ifdef CONFIG_PCI int esc_mods_dma_map_memory(struct file *fp, struct MODS_DMA_MAP_MEMORY *p) { - struct MODS_MEM_INFO *p_mem_info; - struct MODS_MAP_CHUNK *p_map_chunk; - struct pci_dev *p_pci_dev; - unsigned int devfn = PCI_DEVFN(p->pci_device.device, - p->pci_device.function); + struct MODS_MEM_INFO *p_mem_info; + struct MODS_DMA_MAP *p_dma_map; + struct pci_dev *dev = NULL; + int err = -EINVAL; LOG_ENT(); p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; - p_pci_dev = MODS_PCI_GET_SLOT(p->pci_device.domain, - p->pci_device.bus, - devfn); + if (unlikely(!p_mem_info)) { + mods_error_printk("no allocation given\n"); + LOG_EXT(); + return -EINVAL; + } - if (!p_pci_dev) { - mods_error_printk("pci device %04x:%02x:%02x.%x not found\n", + if (mods_is_pci_dev(p_mem_info->dev, &p->pci_device)) { + mods_debug_printk(DEBUG_MEM_DETAILED, + "memory %p already mapped to dev %04x:%02x:%02x.%x\n", + p_mem_info, p->pci_device.domain, p->pci_device.bus, p->pci_device.device, p->pci_device.function); - return -EINVAL; - } - - p_map_chunk = mods_find_dma_map_chunk(p_mem_info, p_pci_dev, NULL); - if (p_map_chunk) { - mods_debug_printk(DEBUG_MEM_DETAILED, - "memory %p already mapped to dev %04x:%02x:%02x.%x\n", - p_mem_info, - p->pci_device.domain, - p->pci_device.bus, - p->pci_device.device, - p->pci_device.function); LOG_EXT(); return 0; } - if (mods_create_dma_map(p_mem_info, p_pci_dev)) { - mods_error_printk("failed to create dma map\n"); - return -ENOMEM; + p_dma_map = find_dma_map(p_mem_info, &p->pci_device); + if (p_dma_map) { + mods_debug_printk(DEBUG_MEM_DETAILED, + "memory %p already mapped to dev %04x:%02x:%02x.%x\n", + p_mem_info, + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + LOG_EXT(); + return 0; } + err = mods_find_pci_dev(fp, &p->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + LOG_EXT(); + return err; + } + + err = mods_create_dma_map(p_mem_info, dev); + + pci_dev_put(dev); LOG_EXT(); - return 0; + return err; } int esc_mods_dma_unmap_memory(struct file *fp, struct MODS_DMA_MAP_MEMORY *p) { - struct MODS_MEM_INFO *p_mem_info; - struct pci_dev *p_pci_dev; - unsigned int devfn = PCI_DEVFN(p->pci_device.device, - p->pci_device.function); - int ret; + struct MODS_MEM_INFO *p_mem_info; + struct pci_dev *dev = NULL; + int err = -EINVAL; LOG_ENT(); p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; - p_pci_dev = MODS_PCI_GET_SLOT(p->pci_device.domain, - p->pci_device.bus, - devfn); - - if (!p_pci_dev) { - mods_error_printk("pci device %04x:%02x:%02x.%x not found\n", - p->pci_device.domain, - p->pci_device.bus, - p->pci_device.device, - p->pci_device.function); + if (unlikely(!p_mem_info)) { + mods_error_printk("no allocation given\n"); + LOG_EXT(); return -EINVAL; } - ret = mods_dma_unmap_all(p_mem_info, p_pci_dev); + err = mods_find_pci_dev(fp, &p->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + } else + err = mods_dma_unmap_all(p_mem_info, dev); + + pci_dev_put(dev); LOG_EXT(); - return ret; + return err; } +#endif #ifdef MODS_TEGRA @@ -1340,15 +1516,12 @@ static void clear_entry_cache_mappings if (p_mem_info->cache_type != MODS_MEMORY_CACHED) return; - for (i = 0; i < p_mem_info->max_chunks; i++) { - struct MODS_PHYS_CHUNK *pt = &p_mem_info->pages[i]; + for (i = 0; i < p_mem_info->num_chunks; i++) { + struct MODS_PHYS_CHUNK *chunk = &p_mem_info->pages[i]; u32 chunk_offs = 0; - u32 chunk_offs_end = PAGE_SIZE << pt->order; + u32 chunk_offs_end = PAGE_SIZE << chunk->order; u64 cur_vo_end = cur_vo + chunk_offs_end; - if (!pt->allocated) - break; - if (virt_offs_end <= cur_vo) break; @@ -1370,9 +1543,9 @@ static void clear_entry_cache_mappings u32 i_page = chunk_offs >> PAGE_SHIFT; u32 page_offs = chunk_offs - (i_page << PAGE_SHIFT); u64 page_va = - (u64)(size_t)kmap(pt->p_page + i_page); + (u64)(size_t)kmap(chunk->p_page + i_page); u64 clear_va = page_va + page_offs; - u64 clear_pa = MODS_DMA_TO_PHYS(pt->dma_addr) + u64 clear_pa = MODS_DMA_TO_PHYS(chunk->dma_addr) + chunk_offs; u32 clear_size = PAGE_SIZE - page_offs; u64 remaining = chunk_offs_end - chunk_offs; @@ -1457,22 +1630,23 @@ int esc_mods_flush_cpu_cache_range(struct file *fp, #endif -static int mods_post_alloc(struct MODS_PHYS_CHUNK *pt, +static int mods_post_alloc(struct MODS_PHYS_CHUNK *chunk, u64 phys_addr, struct MODS_MEM_INFO *p_mem_info) { - u32 num_pages = 1U << pt->order; + u32 num_pages = 1U << chunk->order; u32 i; for (i = 0; i < num_pages; i++) { u64 ptr = 0; - int ret = 0; + int err = 0; + #ifdef CONFIG_BIGPHYS_AREA if (p_mem_info->alloc_type == MODS_ALLOC_TYPE_BIGPHYS_AREA) { ptr = p_mem_info->logical_addr + (i << PAGE_SHIFT); } else #endif - ptr = (u64)(size_t)kmap(pt->p_page + i); + ptr = (u64)(size_t)kmap(chunk->p_page + i); if (!ptr) { mods_error_printk("kmap failed\n"); return -EINVAL; @@ -1482,24 +1656,57 @@ static int mods_post_alloc(struct MODS_PHYS_CHUNK *pt, phys_addr + (i << PAGE_SHIFT), PAGE_SIZE); #else - ret = mods_set_mem_type(ptr, 1, p_mem_info->cache_type); + err = mods_set_mem_type(ptr, 1, p_mem_info->cache_type); #endif #ifdef CONFIG_BIGPHYS_AREA if (p_mem_info->alloc_type != MODS_ALLOC_TYPE_BIGPHYS_AREA) #endif kunmap((void *)(size_t)ptr); - if (ret) { + if (err) { mods_error_printk("set cache type failed\n"); return -EINVAL; } } + + if (p_mem_info->dev) { + u64 dev_addr = pci_map_page(p_mem_info->dev, + chunk->p_page, + 0, + num_pages << PAGE_SHIFT, + DMA_BIDIRECTIONAL); + + if (!dev_addr) { + struct pci_dev *dev = p_mem_info->dev; + + mods_error_printk("failed to map page to device %04x:%02x:%02x.%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + return -EINVAL; + } + + dev_addr = mods_compress_nvlink_addr(p_mem_info->dev, + dev_addr); + chunk->dev_addr = dev_addr; + + mods_debug_printk(DEBUG_MEM_DETAILED, + "auto dma map dev_addr=0x%llx, phys_addr=0x%llx on dev %04x:%02x:%02x.%x\n", + (unsigned long long)dev_addr, + (unsigned long long)chunk->dma_addr, + pci_domain_nr(p_mem_info->dev->bus), + p_mem_info->dev->bus->number, + PCI_SLOT(p_mem_info->dev->devfn), + PCI_FUNC(p_mem_info->dev->devfn)); + } + return 0; } -static void mods_pre_free(struct MODS_PHYS_CHUNK *pt, +static void mods_pre_free(struct MODS_PHYS_CHUNK *chunk, struct MODS_MEM_INFO *p_mem_info) { - u32 num_pages = 1U << pt->order; + u32 num_pages = 1U << chunk->order; u32 i; for (i = 0; i < num_pages; i++) { @@ -1509,7 +1716,7 @@ static void mods_pre_free(struct MODS_PHYS_CHUNK *pt, ptr = p_mem_info->logical_addr + (i << PAGE_SHIFT); else #endif - ptr = (u64)(size_t)kmap(pt->p_page + i); + ptr = (u64)(size_t)kmap(chunk->p_page + i); if (ptr) mods_restore_mem_type(ptr, 1, p_mem_info->cache_type); #ifdef CONFIG_BIGPHYS_AREA @@ -1518,56 +1725,3 @@ static void mods_pre_free(struct MODS_PHYS_CHUNK *pt, kunmap((void *)(size_t)ptr); } } - -/* - * Starting on Power9 systems, DMA addresses for NVLink are no longer - * the same as used over PCIE. - * - * Power9 supports a 56-bit Real Address. This address range is compressed - * when accessed over NvLink to allow the GPU to access all of memory using - * its 47-bit Physical address. - * - * If there is an NPU device present on the system, it implies that NvLink - * sysmem links are present and we need to apply the required address - * conversion for NvLink within the driver. This is intended to be temporary - * to ease the transition to kernel APIs to handle NvLink DMA mappings - * via the NPU device. - * - * Note, a deviation from the documented compression scheme is that the - * upper address bits (i.e. bit 56-63) instead of being set to zero are - * preserved during NvLink address compression so the orignal PCIE DMA - * address can be reconstructed on expansion. These bits can be safely - * ignored on NvLink since they are truncated by the GPU. - */ -static u64 mods_compress_nvlink_addr(struct pci_dev *dev, u64 addr) -{ - u64 addr47 = addr; - /* Note, one key difference from the documented compression scheme - * is that BIT59 used for TCE bypass mode on PCIe is preserved during - * NVLink address compression to allow for the resulting DMA address to - * be used transparently on PCIe. - */ -#if defined(MODS_HAS_PNV_PCI_GET_NPU_DEV) - if (pnv_pci_get_npu_dev(dev, 0)) { - addr47 = addr & (1LLU << 59); - addr47 |= ((addr >> 45) & 0x3) << 43; - addr47 |= ((addr >> 49) & 0x3) << 45; - addr47 |= addr & ((1LLU << 43) - 1); - } -#endif - return addr47; -} - -static u64 mods_expand_nvlink_addr(struct pci_dev *dev, u64 addr47) -{ - u64 addr = addr47; -#if defined(MODS_HAS_PNV_PCI_GET_NPU_DEV) - if (pnv_pci_get_npu_dev(dev, 0)) { - addr = addr47 & ((1LLU << 43) - 1); - addr |= (addr47 & (3ULL << 43)) << 2; - addr |= (addr47 & (3ULL << 45)) << 4; - addr |= addr47 & ~((1ULL << 56) - 1); - } -#endif - return addr; -} diff --git a/drivers/misc/mods/mods_pci.c b/drivers/misc/mods/mods_pci.c index 4cbf7450..35365239 100644 --- a/drivers/misc/mods/mods_pci.c +++ b/drivers/misc/mods/mods_pci.c @@ -26,17 +26,81 @@ #include #endif -/************************ - * PCI ESCAPE FUNCTIONS * - ************************/ +int mods_is_pci_dev(struct pci_dev *dev, + struct mods_pci_dev_2 *pcidev) +{ + unsigned int devfn = PCI_DEVFN(pcidev->device, pcidev->function); -static int mods_find_pci_dev(struct file *pfile, + return dev && + pci_domain_nr(dev->bus) == pcidev->domain && + dev->bus->number == pcidev->bus && + dev->devfn == devfn; +} + +int mods_find_pci_dev(struct file *fp, + struct mods_pci_dev_2 *pcidev, + struct pci_dev **retdev) +{ + struct mods_client *client = fp->private_data; + struct pci_dev *dev; + int err; + + if (unlikely(mutex_lock_interruptible(&client->mtx))) + return -EINTR; + + dev = client->cached_dev; + + if (mods_is_pci_dev(dev, pcidev)) { + *retdev = pci_dev_get(dev); + mutex_unlock(&client->mtx); + return OK; + } + + mutex_unlock(&client->mtx); + + dev = NULL; + +#ifdef MODS_HAS_NEW_ACPI_WALK + dev = pci_get_domain_bus_and_slot(pcidev->domain, + pcidev->bus, + PCI_DEVFN(pcidev->device, + pcidev->function)); +#else + while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev))) + if (mods_is_pci_dev(dev, pcidev)) + break; +#endif + + if (dev) { + if (unlikely(mutex_lock_interruptible(&client->mtx))) { + pci_dev_put(dev); + return -EINTR; + } + + if (dev != client->cached_dev) { + pci_dev_put(client->cached_dev); + client->cached_dev = pci_dev_get(dev); + } + + mutex_unlock(&client->mtx); + + err = OK; + } else + err = -ENODEV; + + *retdev = dev; + return err; +} + +static int find_pci_dev_impl(struct file *pfile, struct MODS_FIND_PCI_DEVICE_2 *p, int enum_non_zero_dom) { struct pci_dev *dev = NULL; int index = -1; + LOG_ENT(); + mods_debug_printk(DEBUG_PCI, "find pci dev %04x:%04x, index %d\n", (int) p->vendor_id, @@ -45,8 +109,10 @@ static int mods_find_pci_dev(struct file *pfile, do { dev = pci_get_device(p->vendor_id, p->device_id, dev); - if (!dev) + if (!dev) { + LOG_EXT(); return -EINVAL; + } if (enum_non_zero_dom || !pci_domain_nr(dev->bus)) ++index; @@ -57,34 +123,36 @@ static int mods_find_pci_dev(struct file *pfile, p->pci_device.device = PCI_SLOT(dev->devfn); p->pci_device.function = PCI_FUNC(dev->devfn); + pci_dev_put(dev); + LOG_EXT(); return OK; } int esc_mods_find_pci_dev_2(struct file *pfile, struct MODS_FIND_PCI_DEVICE_2 *p) { - return mods_find_pci_dev(pfile, p, 1); + return find_pci_dev_impl(pfile, p, 1); } int esc_mods_find_pci_dev(struct file *pfile, struct MODS_FIND_PCI_DEVICE *p) { struct MODS_FIND_PCI_DEVICE_2 p2; - int ret; + int err; p2.device_id = p->device_id; p2.vendor_id = p->vendor_id; p2.index = p->index; - ret = mods_find_pci_dev(pfile, &p2, 0); + err = find_pci_dev_impl(pfile, &p2, 0); - if (!ret) { + if (!err) { p->bus_number = p2.pci_device.bus; p->device_number = p2.pci_device.device; p->function_number = p2.pci_device.function; } - return ret; + return err; } static int mods_find_pci_class_code(struct file *pfile, @@ -94,13 +162,17 @@ static int mods_find_pci_class_code(struct file *pfile, struct pci_dev *dev = NULL; int index = -1; + LOG_ENT(); + mods_debug_printk(DEBUG_PCI, "find pci class code %04x, index %d\n", - (int) p->class_code, (int) p->index); + (unsigned int)p->class_code, (int)p->index); do { dev = pci_get_class(p->class_code, dev); - if (!dev) + if (!dev) { + LOG_EXT(); return -EINVAL; + } if (enum_non_zero_dom || !pci_domain_nr(dev->bus)) ++index; @@ -111,6 +183,8 @@ static int mods_find_pci_class_code(struct file *pfile, p->pci_device.device = PCI_SLOT(dev->devfn); p->pci_device.function = PCI_FUNC(dev->devfn); + pci_dev_put(dev); + LOG_EXT(); return OK; } @@ -124,58 +198,66 @@ int esc_mods_find_pci_class_code(struct file *pfile, struct MODS_FIND_PCI_CLASS_CODE *p) { struct MODS_FIND_PCI_CLASS_CODE_2 p2; - int ret; + int err; p2.class_code = p->class_code; p2.index = p->index; - ret = mods_find_pci_class_code(pfile, &p2, 0); + err = mods_find_pci_class_code(pfile, &p2, 0); - if (!ret) { + if (!err) { p->bus_number = p2.pci_device.bus; p->device_number = p2.pci_device.device; p->function_number = p2.pci_device.function; } - return ret; + return err; } int esc_mods_pci_get_bar_info_2(struct file *pfile, struct MODS_PCI_GET_BAR_INFO_2 *p) { struct pci_dev *dev; - unsigned int devfn, bar_resource_offset, i; + unsigned int bar_resource_offset; + unsigned int i; + int err; #if !defined(MODS_HAS_IORESOURCE_MEM_64) __u32 temp; #endif - devfn = PCI_DEVFN(p->pci_device.device, p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, p->pci_device.bus, devfn); + LOG_ENT(); - if (dev == NULL) - return -EINVAL; + err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + if (unlikely(err)) { + LOG_EXT(); + return err; + } mods_debug_printk(DEBUG_PCI, "pci get bar info %04x:%02x:%02x:%x, bar index %d\n", - (int) p->pci_device.domain, - (int) p->pci_device.bus, (int) p->pci_device.device, - (int) p->pci_device.function, (int) p->bar_index); + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function, + p->bar_index); #if defined(CONFIG_PPC64) if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex()))) { + pci_dev_put(dev); LOG_EXT(); return -EINTR; } /* Enable device on the PCI bus */ if (mods_enable_device(pfile->private_data, dev) == 0) { - mods_error_printk( - "unable to enable dev %04x:%02x:%02x.%x\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function); + mods_error_printk("unable to enable dev %04x:%02x:%02x.%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); mutex_unlock(mods_get_irq_mutex()); + pci_dev_put(dev); + LOG_EXT(); return -EINVAL; } @@ -202,89 +284,102 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, p->base_address = pci_resource_start(dev, bar_resource_offset); p->bar_size = pci_resource_len(dev, bar_resource_offset); + pci_dev_put(dev); + LOG_EXT(); return OK; } int esc_mods_pci_get_bar_info(struct file *pfile, struct MODS_PCI_GET_BAR_INFO *p) { - int retval; + int err; struct MODS_PCI_GET_BAR_INFO_2 get_bar_info = { {0} }; - get_bar_info.pci_device.domain = 0; - get_bar_info.pci_device.bus = p->pci_device.bus; - get_bar_info.pci_device.device = p->pci_device.device; - get_bar_info.pci_device.function = p->pci_device.function; - get_bar_info.bar_index = p->bar_index; + get_bar_info.pci_device.domain = 0; + get_bar_info.pci_device.bus = p->pci_device.bus; + get_bar_info.pci_device.device = p->pci_device.device; + get_bar_info.pci_device.function = p->pci_device.function; + get_bar_info.bar_index = p->bar_index; - retval = esc_mods_pci_get_bar_info_2(pfile, &get_bar_info); - if (retval) - return retval; + err = esc_mods_pci_get_bar_info_2(pfile, &get_bar_info); - p->base_address = get_bar_info.base_address; - p->bar_size = get_bar_info.bar_size; - return OK; + if (likely(!err)) { + p->base_address = get_bar_info.base_address; + p->bar_size = get_bar_info.bar_size; + } + + return err; } int esc_mods_pci_get_irq_2(struct file *pfile, struct MODS_PCI_GET_IRQ_2 *p) { struct pci_dev *dev; - unsigned int devfn; + int err; - devfn = PCI_DEVFN(p->pci_device.device, p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, p->pci_device.bus, devfn); + LOG_ENT(); - if (dev == NULL) - return -EINVAL; + err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + if (unlikely(err)) { + LOG_EXT(); + return err; + } mods_debug_printk(DEBUG_PCI, - "pci get irq %04x:%02x:%02x:%x\n", - (int) p->pci_device.domain, - (int) p->pci_device.bus, (int) p->pci_device.device, - (int) p->pci_device.function); + "pci get irq %04x:%02x:%02x:%x irq=%u\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function, + dev->irq); p->irq = dev->irq; + pci_dev_put(dev); + LOG_EXT(); return OK; } int esc_mods_pci_get_irq(struct file *pfile, struct MODS_PCI_GET_IRQ *p) { - int retval; + int err; struct MODS_PCI_GET_IRQ_2 get_irq = { {0} }; - get_irq.pci_device.domain = 0; - get_irq.pci_device.bus = p->pci_device.bus; - get_irq.pci_device.device = p->pci_device.device; - get_irq.pci_device.function = p->pci_device.function; + get_irq.pci_device.domain = 0; + get_irq.pci_device.bus = p->pci_device.bus; + get_irq.pci_device.device = p->pci_device.device; + get_irq.pci_device.function = p->pci_device.function; - retval = esc_mods_pci_get_irq_2(pfile, &get_irq); - if (retval) - return retval; + err = esc_mods_pci_get_irq_2(pfile, &get_irq); - p->irq = get_irq.irq; - return OK; + if (likely(!err)) + p->irq = get_irq.irq; + + return err; } int esc_mods_pci_read_2(struct file *pfile, struct MODS_PCI_READ_2 *p) { struct pci_dev *dev; - unsigned int devfn; + int err; - devfn = PCI_DEVFN(p->pci_device.device, p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, p->pci_device.bus, devfn); + LOG_ENT(); - if (dev == NULL) - return -EINVAL; + err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + if (unlikely(err)) { + LOG_EXT(); + return err; + } mods_debug_printk(DEBUG_PCI, "pci read %04x:%02x:%02x.%x, addr 0x%04x, size %d\n", - (int) p->pci_device.domain, - (int) p->pci_device.bus, (int) p->pci_device.device, - (int) p->pci_device.function, (int) p->address, - (int) p->data_size); + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function, + p->address, + p->data_size); p->data = 0; switch (p->data_size) { @@ -306,14 +401,18 @@ int esc_mods_pci_read_2(struct file *pfile, struct MODS_PCI_READ_2 *p) pci_read_config_dword(dev, p->address, (u32 *) &p->data); break; default: - return -EINVAL; + err = -EINVAL; + break; } - return OK; + + pci_dev_put(dev); + LOG_EXT(); + return err; } int esc_mods_pci_read(struct file *pfile, struct MODS_PCI_READ *p) { - int retval; + int err; struct MODS_PCI_READ_2 pci_read = { {0} }; pci_read.pci_device.domain = 0; @@ -323,39 +422,41 @@ int esc_mods_pci_read(struct file *pfile, struct MODS_PCI_READ *p) pci_read.address = p->address; pci_read.data_size = p->data_size; - retval = esc_mods_pci_read_2(pfile, &pci_read); - if (retval) - return retval; + err = esc_mods_pci_read_2(pfile, &pci_read); - p->data = pci_read.data; - return OK; + if (likely(!err)) + p->data = pci_read.data; + + return err; } int esc_mods_pci_write_2(struct file *pfile, struct MODS_PCI_WRITE_2 *p) { struct pci_dev *dev; - unsigned int devfn; + int err; + + LOG_ENT(); mods_debug_printk(DEBUG_PCI, - "pci write %04x:%02x:%02x.%x, addr 0x%04x, size %d, data 0x%x\n", - (int) p->pci_device.domain, - (int) p->pci_device.bus, (int) p->pci_device.device, - (int) p->pci_device.function, - (int) p->address, (int) p->data_size, (int) p->data); + "pci write %04x:%02x:%02x.%x, addr 0x%04x, size %d, data 0x%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function, + p->address, + p->data_size, + p->data); - devfn = PCI_DEVFN(p->pci_device.device, p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, p->pci_device.bus, devfn); - - if (dev == NULL) { - mods_error_printk( - "pci write to %04x:%02x:%02x.%x, addr 0x%04x, size %d failed\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function, - (unsigned int)p->address, - (int)p->data_size); - return -EINVAL; + err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + LOG_EXT(); + return err; } switch (p->data_size) { @@ -369,9 +470,13 @@ int esc_mods_pci_write_2(struct file *pfile, struct MODS_PCI_WRITE_2 *p) pci_write_config_dword(dev, p->address, p->data); break; default: - return -EINVAL; + err = -EINVAL; + break; } - return OK; + + pci_dev_put(dev); + LOG_EXT(); + return err; } int esc_mods_pci_write(struct file *pfile, @@ -393,19 +498,32 @@ int esc_mods_pci_write(struct file *pfile, int esc_mods_pci_bus_add_dev(struct file *pfile, struct MODS_PCI_BUS_ADD_DEVICES *scan) { -#if defined(CONFIG_PCI) - mods_info_printk("scanning pci bus %x\n", scan->bus); + struct pci_bus *bus; + int err = OK; - /* initiate a PCI bus scan to find hotplugged PCI devices in domain 0 */ - pci_scan_child_bus(pci_find_bus(0, scan->bus)); + LOG_ENT(); - /* add newly found devices */ - pci_bus_add_devices(pci_find_bus(0, scan->bus)); + mods_info_printk("scanning pci bus %02x\n", scan->bus); - return OK; -#else - return -EINVAL; -#endif + bus = pci_find_bus(0, scan->bus); + + if (likely(bus)) { + + /* initiate a PCI bus scan to find hotplugged PCI devices + * in domain 0 + */ + pci_scan_child_bus(bus); + + /* add newly found devices */ + pci_bus_add_devices(bus); + } else { + mods_error_printk("bus %02x not found\n", scan->bus); + err = -EINVAL; + } + + LOG_EXT(); + + return err; } int esc_mods_pci_hot_reset(struct file *pfile, @@ -413,52 +531,50 @@ int esc_mods_pci_hot_reset(struct file *pfile, { #if defined(CONFIG_PPC64) struct pci_dev *dev; - unsigned int devfn; - int retval; + int err; + + LOG_ENT(); mods_debug_printk(DEBUG_PCI, "pci_hot_reset %04x:%02x:%02x.%x\n", - (int) p->pci_device.domain, - (int) p->pci_device.bus, - (int) p->pci_device.device, - (int) p->pci_device.function); + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); - devfn = PCI_DEVFN(p->pci_device.device, p->pci_device.function); - dev = MODS_PCI_GET_SLOT(p->pci_device.domain, p->pci_device.bus, devfn); - - if (dev == NULL) { - mods_error_printk( - "pci_hot_reset cannot find pci device %04x:%02x:%02x.%x\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function); - return -EINVAL; + err = mods_find_pci_dev(pfile, &p->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("pci_hot_reset cannot find pci device %04x:%02x:%02x.%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + LOG_EXT(); + return err; } - retval = pci_set_pcie_reset_state(dev, pcie_hot_reset); - if (retval) { - mods_error_printk( - "pci_hot_reset failed on %04x:%02x:%02x.%x\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function); - return retval; + err = pci_set_pcie_reset_state(dev, pcie_hot_reset); + if (unlikely(err)) + mods_error_printk("pci_hot_reset failed on %04x:%02x:%02x.%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); + else { + + err = pci_set_pcie_reset_state(dev, pcie_deassert_reset); + if (unlikely(err)) + mods_error_printk("pci_hot_reset deassert failed on %04x:%02x:%02x.%x\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); } - retval = pci_set_pcie_reset_state(dev, pcie_deassert_reset); - if (retval) { - mods_error_printk( - "pci_hot_reset deassert failed on %04x:%02x:%02x.%x\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function); - return retval; - } - - return OK; + pci_dev_put(dev); + LOG_EXT(); + return err; #else return -EINVAL; #endif @@ -512,24 +628,25 @@ int esc_mods_device_numa_info_2(struct file *fp, struct MODS_DEVICE_NUMA_INFO_2 *p) { #ifdef MODS_HAS_DEV_TO_NUMA_NODE - unsigned int devfn = PCI_DEVFN(p->pci_device.device, - p->pci_device.function); - struct pci_dev *dev = MODS_PCI_GET_SLOT(p->pci_device.domain, - p->pci_device.bus, devfn); + struct pci_dev *dev; + int err; LOG_ENT(); - if (dev == NULL) { - mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", - p->pci_device.domain, - p->pci_device.bus, p->pci_device.device, - p->pci_device.function); + err = mods_find_pci_dev(fp, &p->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + p->pci_device.domain, + p->pci_device.bus, + p->pci_device.device, + p->pci_device.function); LOG_EXT(); - return -EINVAL; + return err; } p->node = dev_to_node(&dev->dev); - if (-1 != p->node) { + if (p->node != -1) { const unsigned long *maskp = cpumask_bits(cpumask_of_node(p->node)); unsigned int i, word, bit, maskidx; @@ -537,6 +654,7 @@ int esc_mods_device_numa_info_2(struct file *fp, if (((nr_cpumask_bits + 31) / 32) > MAX_CPU_MASKS) { mods_error_printk("too many CPUs (%d) for mask bits\n", nr_cpumask_bits); + pci_dev_put(dev); LOG_EXT(); return -EINVAL; } @@ -553,6 +671,7 @@ int esc_mods_device_numa_info_2(struct file *fp, p->node_count = num_possible_nodes(); p->cpu_count = num_possible_cpus(); + pci_dev_put(dev); LOG_EXT(); return OK; #else @@ -563,7 +682,8 @@ int esc_mods_device_numa_info_2(struct file *fp, int esc_mods_device_numa_info(struct file *fp, struct MODS_DEVICE_NUMA_INFO *p) { - int retval, i; + int err; + int i; struct MODS_DEVICE_NUMA_INFO_2 numa_info = { {0} }; numa_info.pci_device.domain = 0; @@ -571,16 +691,17 @@ int esc_mods_device_numa_info(struct file *fp, numa_info.pci_device.device = p->pci_device.device; numa_info.pci_device.function = p->pci_device.function; - retval = esc_mods_device_numa_info_2(fp, &numa_info); - if (retval) - return retval; + err = esc_mods_device_numa_info_2(fp, &numa_info); - p->node = numa_info.node; - p->node_count = numa_info.node_count; - for (i = 0; i < MAX_CPU_MASKS; i++) - p->node_cpu_mask[i] = numa_info.node_cpu_mask[i]; - p->cpu_count = numa_info.cpu_count; - return OK; + if (likely(!err)) { + p->node = numa_info.node; + p->node_count = numa_info.node_count; + p->cpu_count = numa_info.cpu_count; + for (i = 0; i < MAX_CPU_MASKS; i++) + p->node_cpu_mask[i] = numa_info.node_cpu_mask[i]; + } + + return err; } int esc_mods_get_iommu_state(struct file *pfile, @@ -602,13 +723,19 @@ int esc_mods_get_iommu_state_2(struct file *pfile, #elif defined(MODS_HAS_DMA_OPS) && \ (defined(MODS_HAS_NONCOH_DMA_OPS) || defined(MODS_HAS_MAP_SG_ATTRS)) - unsigned int devfn = PCI_DEVFN(state->pci_device.device, - state->pci_device.function); - struct pci_dev *dev = MODS_PCI_GET_SLOT(state->pci_device.domain, - state->pci_device.bus, - devfn); + const struct dma_map_ops *ops; + struct pci_dev *dev; + int err; - const struct dma_map_ops *ops = get_dma_ops(&dev->dev); + LOG_ENT(); + + err = mods_find_pci_dev(pfile, &state->pci_device, &dev); + if (unlikely(err)) { + LOG_EXT(); + return err; + } + + ops = get_dma_ops(&dev->dev); #if defined(MODS_HAS_NONCOH_DMA_OPS) state->state = (ops != &noncoherent_swiotlb_dma_ops && @@ -618,6 +745,9 @@ int esc_mods_get_iommu_state_2(struct file *pfile, state->state = ops->map_sg != swiotlb_map_sg_attrs ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE; #endif + pci_dev_put(dev); + LOG_EXT(); + #elif defined(CONFIG_PPC64) || defined(CONFIG_ARM64) /* No way to detect, assume SW I/O TLB is disabled on ppc64/arm64 */ state->state = MODS_SWIOTLB_DISABLED; @@ -632,25 +762,40 @@ int esc_mods_pci_set_dma_mask(struct file *file, struct MODS_PCI_DMA_MASK *dma_mask) { int err; - unsigned int devfn = PCI_DEVFN(dma_mask->pci_device.device, - dma_mask->pci_device.function); - struct pci_dev *dev = MODS_PCI_GET_SLOT(dma_mask->pci_device.domain, - dma_mask->pci_device.bus, - devfn); + struct pci_dev *dev; u64 mask; - if (dma_mask->num_bits > 64) + LOG_ENT(); + + if (unlikely(dma_mask->num_bits > 64)) { + mods_error_printk("num_bits=%u exceeds 64\n", + dma_mask->num_bits); + LOG_EXT(); return -EINVAL; + } + + err = mods_find_pci_dev(file, &dma_mask->pci_device, &dev); + if (unlikely(err)) { + if (err == -ENODEV) + mods_error_printk("PCI device %04x:%02x:%02x.%x not found\n", + dma_mask->pci_device.domain, + dma_mask->pci_device.bus, + dma_mask->pci_device.device, + dma_mask->pci_device.function); + LOG_EXT(); + return err; + } + mask = dma_mask->num_bits == 64 ? ~0ULL : (1ULL<num_bits)-1; err = pci_set_dma_mask(dev, mask); if (err) { mods_error_printk("failed to set dma mask 0x%llx for dev %04x:%02x:%02x.%x\n", mask, - (unsigned int)dma_mask->pci_device.domain, - (unsigned int)dma_mask->pci_device.bus, - (unsigned int)dma_mask->pci_device.device, - (unsigned int)dma_mask->pci_device.function); + dma_mask->pci_device.domain, + dma_mask->pci_device.bus, + dma_mask->pci_device.device, + dma_mask->pci_device.function); #if defined(CONFIG_PPC64) /* Ignore error if TCE bypass is on */ if (dev->dma_mask == ~0ULL) @@ -659,5 +804,7 @@ int esc_mods_pci_set_dma_mask(struct file *file, } else err = pci_set_consistent_dma_mask(dev, mask); + pci_dev_put(dev); + LOG_EXT(); return err; }