diff --git a/drivers/misc/mods/mods.h b/drivers/misc/mods/mods.h index 3917dfbf..deb9eced 100644 --- a/drivers/misc/mods/mods.h +++ b/drivers/misc/mods/mods.h @@ -24,7 +24,7 @@ /* Driver version */ #define MODS_DRIVER_VERSION_MAJOR 3 -#define MODS_DRIVER_VERSION_MINOR 83 +#define MODS_DRIVER_VERSION_MINOR 86 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \ ((MODS_DRIVER_VERSION_MINOR/10) << 4) | \ (MODS_DRIVER_VERSION_MINOR%10)) @@ -113,6 +113,17 @@ struct MODS_GET_PHYSICAL_ADDRESS_2 { __u64 physical_address; }; +/* MODS_ESC_GET_PHYSICAL_ADDRESS_3 */ +struct MODS_GET_PHYSICAL_ADDRESS_3 { + /* IN */ + __u64 memory_handle; + __u64 offset; + struct mods_pci_dev_2 pci_device; + + /* OUT */ + __u64 physical_address; +}; + /* MODS_ESC_VIRTUAL_TO_PHYSICAL */ struct MODS_VIRTUAL_TO_PHYSICAL { /* IN */ @@ -393,6 +404,18 @@ struct mods_mask_info2 { __u64 or_mask; /*or mask for setting bit in this register */ }; +struct MODS_REGISTER_IRQ_4 { + /* IN */ + struct mods_pci_dev_2 dev; /* device identifying interrupt for */ + /* which the mask will be applied */ + __u64 aperture_addr; /* physical address of aperture */ + __u32 aperture_size; /* size of the mapped region */ + __u32 mask_info_cnt; /* number of entries in mask_info[]*/ + struct mods_mask_info2 mask_info[MODS_IRQ_MAX_MASKS]; + __u32 irq_count; /* number of irq's to allocate */ + __u32 irq_flags; /* irq type and affinity */ +}; + struct MODS_REGISTER_IRQ_3 { /* IN */ struct mods_pci_dev_2 dev; /* device identifying interrupt for */ @@ -424,6 +447,13 @@ struct MODS_REGISTER_IRQ { __u8 type; /* MODS_IRQ_TYPE_* */ }; +struct mods_irq_3 { + struct mods_pci_dev_2 dev; /* device which generated the interrupt */ + __u32 irq_index; /* index of irq 0 for INTx & MSI */ + __u32 delay; /* delay in ns between the irq */ + /* occurring and MODS querying for it */ +}; + struct mods_irq_2 { __u32 delay; /* delay in ns between the irq */ /* occurring and MODS querying for it */ @@ -440,6 +470,14 @@ struct mods_irq { #define MODS_MAX_IRQS 32 +/* MODS_ESC_QUERY_IRQ_3 */ +struct MODS_QUERY_IRQ_3 { + /* OUT */ + struct mods_irq_3 irq_list[MODS_MAX_IRQS]; + __u8 more; /* indicates that more interrupts */ + /* are waiting */ +}; + /* MODS_ESC_QUERY_IRQ_2 */ struct MODS_QUERY_IRQ_2 { /* OUT */ @@ -458,6 +496,8 @@ struct MODS_QUERY_IRQ { #define MODS_IRQ_TYPE_INT 0 #define MODS_IRQ_TYPE_MSI 1 #define MODS_IRQ_TYPE_CPU 2 +#define MODS_IRQ_TYPE_MSIX 3 +#define MODS_IRQ_TYPE_MASK 0xff /* MODS_ESC_SET_IRQ_MULTIMASK */ struct mods_mask_info { @@ -989,7 +1029,6 @@ struct MODS_GET_NVLINK_LINE_RATE { __u32 speed; }; - /* MODS_ESC_ACQUIRE_ACCESS_TOKEN * MODS_ESC_RELEASE_ACCESS_TOKEN * MODS_ESC_VERIFY_ACCESS_TOKEN @@ -1000,6 +1039,27 @@ struct MODS_ACCESS_TOKEN { __u32 token; }; +#define MODS_MAX_SYSFS_PATH_BUF_SIZE 512 +#define MODS_MAX_SYSFS_PATH_LEN (512 - 6) +#define MODS_MAX_SYSFS_FILE_SIZE 4096 + +/* MODS_ESC_WRITE_SYSFS_NODE */ +struct MODS_SYSFS_NODE { + /* IN */ + char path[MODS_MAX_SYSFS_PATH_BUF_SIZE]; + char contents[MODS_MAX_SYSFS_FILE_SIZE]; + __u32 size; +}; + +#define MODS_IRQ_TYPE_FROM_FLAGS(flags) ((flags)&0xf) + +/* MODS_ESC_SET_NUM_VF */ +struct MODS_SET_NUM_VF { + /* IN */ + struct mods_pci_dev_2 dev; + __u32 numvfs; /* number of virtual functions */ +}; + #pragma pack(pop) /* ************************************************************************* */ @@ -1247,5 +1307,21 @@ struct MODS_ACCESS_TOKEN { _IOW(MODS_IOC_MAGIC, 109, struct MODS_ACCESS_TOKEN) #define MODS_ESC_GET_IOMMU_STATE \ _IOWR(MODS_IOC_MAGIC, 110, struct MODS_GET_IOMMU_STATE) +#define MODS_ESC_WRITE_SYSFS_NODE \ + _IOW(MODS_IOC_MAGIC, 111, struct MODS_SYSFS_NODE) +#define MODS_ESC_GET_PHYSICAL_ADDRESS_2 \ + _IOWR(MODS_IOC_MAGIC, 112, \ + struct MODS_GET_PHYSICAL_ADDRESS_3) +#define MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3 \ + _IOWR(MODS_IOC_MAGIC, 113, \ + struct MODS_GET_PHYSICAL_ADDRESS_3) +#define MODS_ESC_REGISTER_IRQ_4 \ + _IOW(MODS_IOC_MAGIC, 114, struct MODS_REGISTER_IRQ_4) +#define MODS_ESC_QUERY_IRQ_3 \ + _IOR(MODS_IOC_MAGIC, 115, struct MODS_QUERY_IRQ_3) +#define MODS_ESC_SET_NUM_VF \ + _IOW(MODS_IOC_MAGIC, 116, struct MODS_SET_NUM_VF) +#define MODS_ESC_SET_TOTAL_VF \ + _IOW(MODS_IOC_MAGIC, 117, struct MODS_SET_NUM_VF) #endif /* _MODS_H_ */ diff --git a/drivers/misc/mods/mods_config.h b/drivers/misc/mods/mods_config.h index b8b1fbcc..dd0a8a27 100644 --- a/drivers/misc/mods/mods_config.h +++ b/drivers/misc/mods/mods_config.h @@ -58,6 +58,8 @@ #ifdef CONFIG_PCI #define MODS_CAN_REGISTER_PCI_DEV 1 #endif +#define MODS_HAS_MSIX_RANGE 1 +#define MODS_HAS_SRIOV 1 #endif /* _MODS_CONFIG_H_ */ diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h index b6e08c76..d7f5b7f4 100644 --- a/drivers/misc/mods/mods_internal.h +++ b/drivers/misc/mods/mods_internal.h @@ -55,6 +55,10 @@ struct en_dev_entry { struct pci_dev *dev; struct en_dev_entry *next; + __u32 irqs_allocated; + __u32 irq_flags; + __u32 nvecs; + struct msix_entry *msix_entries; }; struct mem_type { @@ -150,7 +154,7 @@ struct SYS_MAP_MEMORY { * machine address on Xen */ u64 virtual_addr; /* virtual address of given mapping */ - u32 mapping_length; /* tells how many bytes were mapped */ + u64 mapping_length; /* tells how many bytes were mapped */ struct list_head list; }; @@ -193,7 +197,7 @@ struct NVL_TRAINED { /* debug print masks */ #define DEBUG_IOCTL 0x2 -#define DEBUG_PCICFG 0x4 +#define DEBUG_PCI 0x4 #define DEBUG_ACPI 0x8 #define DEBUG_ISR 0x10 #define DEBUG_MEM 0x20 @@ -204,7 +208,7 @@ struct NVL_TRAINED { #define DEBUG_TEGRADMA 0x400 #define DEBUG_ISR_DETAILED (DEBUG_ISR | DEBUG_DETAILED) #define DEBUG_MEM_DETAILED (DEBUG_MEM | DEBUG_DETAILED) -#define DEBUG_ALL (DEBUG_IOCTL | DEBUG_PCICFG | DEBUG_ACPI | \ +#define DEBUG_ALL (DEBUG_IOCTL | DEBUG_PCI | DEBUG_ACPI | \ DEBUG_ISR | DEBUG_MEM | DEBUG_FUNC | DEBUG_CLOCK | DEBUG_DETAILED | \ DEBUG_TEGRADC | DEBUG_TEGRADMA) @@ -230,6 +234,7 @@ struct irq_q_data { u32 time; struct pci_dev *dev; u32 irq; + u32 irq_index; }; struct irq_q_info { @@ -250,6 +255,7 @@ struct irq_mask_info { struct dev_irq_map { void *dev_irq_aperture; u32 apic_irq; + u32 entry; u8 type; u8 channel; u8 mask_info_cnt; @@ -388,6 +394,7 @@ int mods_unregister_all_nvlink_sysmem_trained(struct file *fp); #ifdef CONFIG_PCI int mods_enable_device(struct mods_file_private_data *priv, struct pci_dev *pdev); +void mods_disable_device(struct pci_dev *pdev); int mods_unregister_all_pci_res_mappings(struct file *fp); #define MODS_UNREGISTER_PCI_MAP(fp) mods_unregister_all_pci_res_mappings(fp) #else @@ -412,10 +419,14 @@ int esc_mods_free_pages(struct file *fp, struct MODS_FREE_PAGES *p); int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p); int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p); +int esc_mods_get_phys_addr_2(struct file *fp, + struct MODS_GET_PHYSICAL_ADDRESS_3 *p); int esc_mods_get_mapped_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p); int esc_mods_get_mapped_phys_addr_2(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_2 *p); +int esc_mods_get_mapped_phys_addr_3(struct file *fp, + struct MODS_GET_PHYSICAL_ADDRESS_3 *p); int esc_mods_virtual_to_phys(struct file *fp, struct MODS_VIRTUAL_TO_PHYSICAL *p); int esc_mods_phys_to_virtual(struct file *fp, @@ -507,6 +518,11 @@ int esc_mods_set_irq_multimask(struct file *fp, int esc_mods_irq_handled(struct file *fp, struct MODS_REGISTER_IRQ *p); int esc_mods_irq_handled_2(struct file *fp, struct MODS_REGISTER_IRQ_2 *p); + +int esc_mods_register_irq_4(struct file *fp, + struct MODS_REGISTER_IRQ_4 *p); +int esc_mods_query_irq_3(struct file *fp, struct MODS_QUERY_IRQ_3 *p); + #ifdef MODS_TEGRA /* clock */ diff --git a/drivers/misc/mods/mods_irq.c b/drivers/misc/mods/mods_irq.c index 35905430..9a5d72f1 100644 --- a/drivers/misc/mods/mods_irq.c +++ b/drivers/misc/mods/mods_irq.c @@ -73,6 +73,11 @@ int mods_enable_device(struct mods_file_private_data *priv, int ret = -1; struct en_dev_entry *entry = priv->enabled_devices; + mods_debug_printk(DEBUG_ISR_DETAILED, "vendor %x device %x device %x function %x\n", + pdev->vendor, pdev->device, + (unsigned int)PCI_SLOT(pdev->devfn), + (unsigned int)PCI_FUNC(pdev->devfn)); + while (entry != 0) { if (entry->dev == pdev) return 0; @@ -81,15 +86,38 @@ int mods_enable_device(struct mods_file_private_data *priv, ret = pci_enable_device(pdev); if (ret == 0) { - entry = kmalloc(sizeof(*entry), GFP_KERNEL | __GFP_NORETRY); + entry = kzalloc(sizeof(*entry), GFP_KERNEL | __GFP_NORETRY); if (unlikely(!entry)) return 0; entry->dev = pdev; entry->next = priv->enabled_devices; priv->enabled_devices = entry; + pci_set_drvdata(pdev, entry); + mods_debug_printk(DEBUG_ISR_DETAILED, + "pci_set_drvdata for vendor %x device %x device %x function %x\n", + pdev->vendor, pdev->device, + (unsigned int)PCI_SLOT(pdev->devfn), + (unsigned int)PCI_FUNC(pdev->devfn)); + } else { + mods_debug_printk(DEBUG_ISR_DETAILED, + "pci_set_drvdata failed for vendor %x device %x device %x function %x ret=%d\n", + pdev->vendor, pdev->device, + (unsigned int)PCI_SLOT(pdev->devfn), + (unsigned int)PCI_FUNC(pdev->devfn), + ret); } return ret; } + +void mods_disable_device(struct pci_dev *pdev) +{ + struct en_dev_entry *dpriv = pci_get_drvdata(pdev); + + if (dpriv) + pci_set_drvdata(pdev, NULL); + + pci_disable_device(pdev); +} #endif static unsigned int get_cur_time(void) @@ -165,6 +193,22 @@ static void mods_disable_interrupts(struct dev_irq_map *t) } } +static const char *mods_irq_type_name(u8 irq_type) +{ + switch (irq_type) { + case MODS_IRQ_TYPE_INT: + return "INTx"; + case MODS_IRQ_TYPE_MSI: + return "MSI"; + case MODS_IRQ_TYPE_CPU: + return "CPU"; + case MODS_IRQ_TYPE_MSIX: + return "MSI-X"; + default: + return "unknown"; + } +} + static void rec_irq_done(struct nv_device *dev, unsigned char channel, struct dev_irq_map *t, @@ -182,17 +226,12 @@ static void rec_irq_done(struct nv_device *dev, unsigned int i; for (i = q->head; i != q->tail; i++) { - if (t->dev) { - struct pci_dev *cur - = q->data[i & (MODS_MAX_IRQS - 1)].dev; - if (cur == t->dev) - return; - } else { - u32 cur - = q->data[i & (MODS_MAX_IRQS - 1)].irq; - if (cur == t->apic_irq) - return; - } + struct irq_q_data *pd + = q->data+(i & (MODS_MAX_IRQS - 1)); + + if ((pd->irq == t->apic_irq) && + (!t->dev || (pd->dev == t->dev))) + return; } } @@ -204,9 +243,10 @@ static void rec_irq_done(struct nv_device *dev, } /* Record the device which generated the IRQ in the queue */ - q->data[q->tail & (MODS_MAX_IRQS - 1)].dev = t->dev; - q->data[q->tail & (MODS_MAX_IRQS - 1)].irq = t->apic_irq; - q->data[q->tail & (MODS_MAX_IRQS - 1)].time = irq_time; + q->data[q->tail & (MODS_MAX_IRQS - 1)].dev = t->dev; + q->data[q->tail & (MODS_MAX_IRQS - 1)].irq = t->apic_irq; + q->data[q->tail & (MODS_MAX_IRQS - 1)].irq_index = t->entry; + q->data[q->tail & (MODS_MAX_IRQS - 1)].time = irq_time; q->tail++; #ifdef CONFIG_PCI @@ -217,7 +257,7 @@ static void rec_irq_done(struct nv_device *dev, (unsigned int)(t->dev->bus->number), (unsigned int)PCI_SLOT(t->dev->devfn), (unsigned int)PCI_FUNC(t->dev->devfn), - (t->type == MODS_IRQ_TYPE_MSI) ? "MSI" : "INTx", + mods_irq_type_name(t->type), t->apic_irq, irq_time); } else @@ -267,7 +307,7 @@ static irqreturn_t mods_irq_handle(int irq, void *data rec_irq_done(dev, channel_idx+1, t, irq_time); found |= 1; - /* MSI and CPU interrupts are not shared, + /* MSI, MSI-X and CPU interrupts are not shared, * so stop looking */ if (t->type != MODS_IRQ_TYPE_INT) { @@ -299,9 +339,9 @@ static int mods_lookup_irq(unsigned char channel, struct pci_dev *pdev, next, &pmp->irq_head[channel_idx], list) { - if ((pdev && (t->dev == pdev)) - || (!pdev && (t->apic_irq == irq))) { + if ((t->apic_irq == irq) && + (!pdev || (t->dev == pdev))) { if (channel == 0) { ret = IRQ_FOUND; } else { @@ -338,7 +378,7 @@ static int is_nvidia_device(struct pci_dev *pdev) #ifdef CONFIG_PCI static void setup_mask_info(struct dev_irq_map *newmap, - struct MODS_REGISTER_IRQ_3 *p, + struct MODS_REGISTER_IRQ_4 *p, struct pci_dev *pdev) { /* account for legacy adapters */ @@ -372,9 +412,9 @@ static void setup_mask_info(struct dev_irq_map *newmap, static int add_irq_map(unsigned char channel, struct pci_dev *pdev, - u32 irq, - struct MODS_REGISTER_IRQ_3 *p) + struct MODS_REGISTER_IRQ_4 *p, u32 irq, u32 entry) { + u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); struct dev_irq_map *newmap = NULL; struct mods_priv *pmp = get_all_data(); struct nv_device *nvdev = get_dev(); @@ -394,13 +434,14 @@ static int add_irq_map(unsigned char channel, newmap->channel = channel; newmap->dev_irq_aperture = 0; newmap->mask_info_cnt = 0; - newmap->type = p->irq_type; + newmap->type = irq_type; + newmap->entry = entry; /* Enable IRQ for this device in the kernel */ if (request_irq( irq, &mods_irq_handle, - (p->irq_type == MODS_IRQ_TYPE_INT) ? IRQF_SHARED : 0, + (irq_type == MODS_IRQ_TYPE_INT) ? IRQF_SHARED : 0, nvdev->name, nvdev)) { mods_error_printk("unable to enable IRQ 0x%x\n", irq); @@ -414,7 +455,7 @@ static int add_irq_map(unsigned char channel, #ifdef CONFIG_PCI /* Map BAR0 to be able to disable interrupts */ - if ((p->irq_type == MODS_IRQ_TYPE_INT) && + if ((irq_type == MODS_IRQ_TYPE_INT) && (p->aperture_addr != 0) && (p->aperture_size != 0)) { char *bar = ioremap_nocache(p->aperture_addr, p->aperture_size); @@ -433,21 +474,24 @@ static int add_irq_map(unsigned char channel, #endif /* Print out successful registration string */ - if (p->irq_type == MODS_IRQ_TYPE_CPU) + if (irq_type == MODS_IRQ_TYPE_CPU) mods_debug_printk(DEBUG_ISR, "registered CPU IRQ 0x%x\n", irq); #ifdef CONFIG_PCI - else if (p->irq_type == MODS_IRQ_TYPE_INT) { + else if ((irq_type == MODS_IRQ_TYPE_INT) || + (irq_type == MODS_IRQ_TYPE_MSI) || + (irq_type == MODS_IRQ_TYPE_MSIX)) { mods_debug_printk(DEBUG_ISR, - "%04x:%x:%02x.%x registered INTx IRQ 0x%x\n", + "%04x:%x:%02x.%x registered %s IRQ 0x%x\n", (unsigned int)(pci_domain_nr(pdev->bus)), (unsigned int)(pdev->bus->number), (unsigned int)PCI_SLOT(pdev->devfn), (unsigned int)PCI_FUNC(pdev->devfn), - pdev->irq); + mods_irq_type_name(irq_type), + irq); } #endif #ifdef CONFIG_PCI_MSI - else if (p->irq_type == MODS_IRQ_TYPE_MSI) { + else if (irq_type == MODS_IRQ_TYPE_MSI) { u16 control; u16 data; int cap_pos = pci_find_capability(pdev, PCI_CAP_ID_MSI); @@ -467,8 +511,16 @@ static int add_irq_map(unsigned char channel, (unsigned int)(pdev->bus->number), (unsigned int)PCI_SLOT(pdev->devfn), (unsigned int)PCI_FUNC(pdev->devfn), - pdev->irq, + irq, (unsigned int)data); + } else if (irq_type == MODS_IRQ_TYPE_MSIX) { + mods_debug_printk(DEBUG_ISR, + "%04x:%x:%02x.%x registered MSI-X IRQ 0x%x\n", + (unsigned int)(pci_domain_nr(pdev->bus)), + (unsigned int)(pdev->bus->number), + (unsigned int)PCI_SLOT(pdev->devfn), + (unsigned int)PCI_FUNC(pdev->devfn), + irq); } #endif @@ -490,12 +542,6 @@ static void mods_free_map(struct dev_irq_map *del) /* Unhook interrupts in the kernel */ free_irq(del->apic_irq, get_dev()); - /* Disable MSI */ -#ifdef CONFIG_PCI_MSI - if (del->type == MODS_IRQ_TYPE_MSI) - pci_disable_msi(del->dev); -#endif - /* Free memory */ kfree(del); @@ -588,37 +634,88 @@ unsigned char mods_alloc_channel(void) return channel; } -void mods_free_channel(unsigned char channel) +static int mods_free_irqs(unsigned char channel, struct pci_dev *dev) { +#ifdef CONFIG_PCI struct mods_priv *pmp = get_all_data(); struct dev_irq_map *del = NULL; - struct dev_irq_map *next = NULL; + struct dev_irq_map *next; + struct en_dev_entry *dpriv = pci_get_drvdata(dev); + unsigned int irq_type; + + LOG_ENT(); + + if (!dpriv || !dpriv->irqs_allocated) { + LOG_EXT(); + return OK; + } + + mods_debug_printk(DEBUG_ISR_DETAILED, + "(dev=%04x:%x:%02x.%x) irqs_allocated=%d irq_flags=0x%x nvecs=%d\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + dpriv->irqs_allocated, + dpriv->irq_flags, + dpriv->nvecs + ); + + /* Delete device interrupts from the list */ + list_for_each_entry_safe(del, next, &pmp->irq_head[channel - 1], list) { + if (dev == del->dev) { + list_del(&del->list); + mods_debug_printk(DEBUG_ISR, + "%04x:%x:%02x.%x unregistered %s IRQ 0x%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), + mods_irq_type_name(del->type), + del->apic_irq); + mods_free_map(del); + + if (del->type != MODS_IRQ_TYPE_MSIX) + break; + } + } + + mods_debug_printk(DEBUG_ISR_DETAILED, "before disable\n"); +#ifdef CONFIG_PCI_MSI + irq_type = MODS_IRQ_TYPE_FROM_FLAGS(dpriv->irq_flags); + + if (irq_type == MODS_IRQ_TYPE_MSIX) { + pci_disable_msix(dev); + kfree(dpriv->msix_entries); + } else if (irq_type == MODS_IRQ_TYPE_MSI) { + pci_disable_msi(dev); + } +#endif + + dpriv->nvecs = 0; + dpriv->msix_entries = NULL; + dpriv->irqs_allocated = false; + mods_debug_printk(DEBUG_ISR_DETAILED, "irqs freed\n"); +#endif + + LOG_EXT(); + return 0; +} + +void mods_free_channel(unsigned char channel) +{ + struct nv_device *nvdev = get_dev(); + MODS_PRIV priv = nvdev->pri[channel - 1]; + struct en_dev_entry *dent = priv->enabled_devices; + struct mods_priv *pmp = get_all_data(); struct irq_q_info *q = &pmp->rec_info[channel - 1]; LOG_ENT(); /* Release all interrupts */ - list_for_each_entry_safe(del, next, &pmp->irq_head[channel - 1], list) { - list_del(&del->list); - if (del->type == MODS_IRQ_TYPE_CPU) { - mods_warning_printk( - "CPU IRQ 0x%x is still hooked, unhooking\n", - del->apic_irq); - } -#ifdef CONFIG_PCI - else { - mods_warning_printk( - "%s IRQ 0x%x for dev %04x:%x:%02x.%x is still hooked, unhooking\n", - (del->type == MODS_IRQ_TYPE_MSI) - ? "MSI" : "INTx", - del->dev->irq, - (unsigned int)(pci_domain_nr(del->dev->bus)), - (unsigned int)(del->dev->bus->number), - (unsigned int)PCI_SLOT(del->dev->devfn), - (unsigned int)PCI_FUNC(del->dev->devfn)); - } -#endif - mods_free_map(del); + while (dent) { + mods_free_irqs(priv->mods_id, dent->dev); + dent = dent->next; } /* Clear queue */ @@ -631,14 +728,229 @@ void mods_free_channel(unsigned char channel) (unsigned int)channel); LOG_EXT(); } + #ifdef CONFIG_PCI -static int mods_register_pci_irq(struct file *pfile, - struct MODS_REGISTER_IRQ_3 *p) +static int mods_allocate_irqs(struct file *pfile, struct mods_pci_dev_2 *bdf, + __u32 nvecs, __u32 flags) { + MODS_PRIV private_data = pfile->private_data; struct pci_dev *dev; + struct en_dev_entry *dpriv; + unsigned int devfn; + unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(flags); + + LOG_ENT(); + + mods_debug_printk(DEBUG_ISR_DETAILED, + "(dev=%04x:%x:%02x.%x, flags=0x%x, nvecs=%d)\n", + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function, + flags, + nvecs); + + if (!nvecs) { + mods_error_printk("no irq's requested!\n"); + LOG_EXT(); + return -EINVAL; + } + + /* Get the PCI device structure for the specified device from kernel */ + devfn = PCI_DEVFN(bdf->device, bdf->function); + dev = MODS_PCI_GET_SLOT(bdf->domain, bdf->bus, devfn); + if (!dev) { + mods_error_printk( + "unknown dev %04x:%x:%02x.%x\n", + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function); + LOG_EXT(); + return -EINVAL; + } + + /* Determine if the device supports requested interrupt type */ + if (irq_type == MODS_IRQ_TYPE_MSI) { +#ifdef CONFIG_PCI_MSI + if (pci_find_capability(dev, PCI_CAP_ID_MSI) == 0) { + mods_error_printk( + "dev %04x:%x:%02x.%x does not support MSI\n", + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function); + LOG_EXT(); + return -EINVAL; + } +#else + mods_error_printk("the kernel does not support MSI!\n"); + return -EINVAL; +#endif + } else if (irq_type == MODS_IRQ_TYPE_MSIX) { +#ifdef CONFIG_PCI_MSI + int msix_cap; + + msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); + + if (!msix_cap) { + mods_error_printk( + "dev %04x:%x:%02x.%x does not support MSI-X\n", + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function); + LOG_EXT(); + return -EINVAL; + } +#else + mods_error_printk("the kernel does not support MSIX!\n"); + return -EINVAL; +#endif + } + + /* Enable device on the PCI bus */ + if (mods_enable_device(private_data, dev)) { + mods_error_printk("unable to enable dev %04x:%x:%02x.%x\n", + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function); + LOG_EXT(); + return -EINVAL; + } + + dpriv = pci_get_drvdata(dev); + + if (!dpriv) { + mods_error_printk("could not get mods dev private!\n"); + LOG_EXT(); + return -EINVAL; + } + + if (irq_type == MODS_IRQ_TYPE_INT) { + /* use legacy irq */ + if (nvecs != 1) { + mods_error_printk("INTA: only 1 INTA vector supported requested %d!\n", + nvecs); + LOG_EXT(); + return -EINVAL; + } + dpriv->nvecs = 1; + } + /* Enable MSI */ +#ifdef CONFIG_PCI_MSI + else if (irq_type == MODS_IRQ_TYPE_MSI) { + if (nvecs != 1) { + mods_error_printk("MSI: only 1 MSI vector supported requested %d!\n", + nvecs); + LOG_EXT(); + return -EINVAL; + } + if (pci_enable_msi(dev) != 0) { + mods_error_printk( + "unable to enable MSI on dev %04x:%x:%02x.%x\n", + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function); + return -EINVAL; + } + dpriv->nvecs = 1; + } else if (irq_type == MODS_IRQ_TYPE_MSIX) { + struct msix_entry *entries; + int i = 0, cnt = 1; + + entries = kcalloc(nvecs, sizeof(struct msix_entry), + GFP_KERNEL | __GFP_NORETRY); + + if (!entries) { + mods_error_printk("could not allocate %d MSI-X entries!\n", + nvecs); + LOG_EXT(); + return -ENOMEM; + } + + for (i = 0; i < nvecs; i++) + entries[i].entry = (uint16_t)i; + +#ifdef MODS_HAS_MSIX_RANGE + cnt = pci_enable_msix_range(dev, entries, nvecs, nvecs); + + if (cnt < 0) { + /* returns number of interrupts allocated + * < 0 indicates a failure. + */ + mods_error_printk( + "could not allocate the requested number of MSI-X vectors=%d return=%d!\n", + nvecs, cnt); + kfree(entries); + LOG_EXT(); + return cnt; + } +#else + cnt = pci_enable_msix(dev, entries, nvecs); + + if (cnt) { + /* A return of < 0 indicates a failure. + * A return of > 0 indicates that driver request is + * exceeding the number of irqs or MSI-X + * vectors available + */ + mods_error_printk( + "could not allocate the requested number of MSI-X vectors=%d return=%d!\n", + nvecs, cnt); + kfree(entries); + LOG_EXT(); + if (cnt > 0) + cnt = -ENOSPC; + return cnt; + } +#endif + + mods_debug_printk(DEBUG_ISR, + "allocated %d irq's of type %s(%d)\n", + nvecs, mods_irq_type_name(irq_type), irq_type); + + for (i = 0; i < nvecs; i++) + mods_debug_printk(DEBUG_ISR, "vec %d %x\n", + entries[i].entry, entries[i].vector); + + dpriv->nvecs = nvecs; + dpriv->msix_entries = entries; + } +#endif + else { + mods_error_printk( + "unsupported irq_type %d dev: %04x:%x:%02x.%x\n", + irq_type, + (unsigned int)bdf->domain, + (unsigned int)bdf->bus, + (unsigned int)bdf->device, + (unsigned int)bdf->function); + LOG_EXT(); + return -EINVAL; + } + + dpriv->irqs_allocated = true; + dpriv->irq_flags = flags; + LOG_EXT(); + return OK; +} + +static int mods_unregister_pci_irq(struct file *pfile, + struct MODS_REGISTER_IRQ_2 *p); + +static int mods_register_pci_irq(struct file *pfile, + struct MODS_REGISTER_IRQ_4 *p) +{ + int rc = OK; + unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); + struct pci_dev *dev; + struct en_dev_entry *dpriv; unsigned int devfn; unsigned char channel; - MODS_PRIV private_data = pfile->private_data; + int i; LOG_ENT(); @@ -654,14 +966,31 @@ static int mods_register_pci_irq(struct file *pfile, devfn = PCI_DEVFN(p->dev.device, p->dev.function); dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); if (!dev) { + mods_error_printk( + "unknown dev %04x:%x:%02x.%x\n", + (unsigned int)p->dev.domain, + (unsigned int)p->dev.bus, + (unsigned int)p->dev.device, + (unsigned int)p->dev.function); LOG_EXT(); return -EINVAL; } - /* Determine if the interrupt is already hooked */ - if (mods_lookup_irq(0, dev, 0) == IRQ_FOUND) { + dpriv = pci_get_drvdata(dev); + + if (!dpriv || !dpriv->irqs_allocated) { + if (mods_allocate_irqs(pfile, &p->dev, p->irq_count, + p->irq_flags)) { + mods_error_printk( + "could not allocate irqs for irq_type %d\n", + irq_type); + LOG_EXT(); + return -EINVAL; + } + dpriv = pci_get_drvdata(dev); + } else { mods_error_printk( - "IRQ for dev %04x:%x:%02x.%x has already been registered\n", + "dev %04x:%x:%02x.%x has already been registered\n", (unsigned int)p->dev.domain, (unsigned int)p->dev.bus, (unsigned int)p->dev.device, @@ -670,75 +999,36 @@ static int mods_register_pci_irq(struct file *pfile, return -EINVAL; } - /* Determine if the device supports MSI */ - if (p->irq_type == MODS_IRQ_TYPE_MSI) { + for (i = 0; i < p->irq_count; i++) { + u32 irq = ((irq_type == MODS_IRQ_TYPE_INT) || + (irq_type == MODS_IRQ_TYPE_MSI)) ? dev->irq : + dpriv->msix_entries[i].vector; + + if (add_irq_map(channel, dev, p, irq, i) != OK) { #ifdef CONFIG_PCI_MSI - if (pci_find_capability(dev, PCI_CAP_ID_MSI) == 0) { - mods_error_printk( - "dev %04x:%x:%02x.%x does not support MSI\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); + if (irq_type == MODS_IRQ_TYPE_MSI) + pci_disable_msi(dev); + else if (irq_type == MODS_IRQ_TYPE_MSIX) + pci_disable_msix(dev); +#endif LOG_EXT(); return -EINVAL; } -#else - mods_error_printk("the kernel does not support MSI!\n"); - return -EINVAL; -#endif } - /* Enable device on the PCI bus */ - if (mods_enable_device(private_data, dev)) { - mods_error_printk("unable to enable dev %04x:%x:%02x.%x\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); - LOG_EXT(); - return -EINVAL; - } - - /* Enable MSI */ -#ifdef CONFIG_PCI_MSI - if (p->irq_type == MODS_IRQ_TYPE_MSI) { - if (pci_enable_msi(dev) != 0) { - mods_error_printk( - "unable to enable MSI on dev %04x:%x:%02x.%x\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); - return -EINVAL; - } - } -#endif - - /* Register interrupt */ - if (add_irq_map(channel, dev, dev->irq, p) != OK) { -#ifdef CONFIG_PCI_MSI - if (p->irq_type == MODS_IRQ_TYPE_MSI) - pci_disable_msi(dev); -#endif - LOG_EXT(); - return -EINVAL; - } - - return OK; + LOG_EXT(); + return rc; } #endif /* CONFIG_PCI */ static int mods_register_cpu_irq(struct file *pfile, - struct MODS_REGISTER_IRQ_3 *p) + struct MODS_REGISTER_IRQ_4 *p) { unsigned char channel; - unsigned int irq; + u32 irq = p->dev.bus; LOG_ENT(); - irq = p->dev.bus; - /* Identify the caller */ channel = MODS_GET_FILE_PRIVATE_ID(pfile); WARN_ON(id_is_valid(channel) != OK); @@ -756,7 +1046,7 @@ static int mods_register_cpu_irq(struct file *pfile, } /* Register interrupt */ - if (add_irq_map(channel, 0, irq, p) != OK) { + if (add_irq_map(channel, 0, p, irq, 0) != OK) { LOG_EXT(); return -EINVAL; } @@ -768,12 +1058,10 @@ static int mods_register_cpu_irq(struct file *pfile, static int mods_unregister_pci_irq(struct file *pfile, struct MODS_REGISTER_IRQ_2 *p) { - struct mods_priv *pmp = get_all_data(); - struct dev_irq_map *del = NULL; - struct dev_irq_map *next; struct pci_dev *dev; unsigned int devfn; unsigned char channel; + int rv = OK; LOG_ENT(); @@ -793,43 +1081,10 @@ static int mods_unregister_pci_irq(struct file *pfile, return -EINVAL; } - /* Determine if the interrupt is already hooked by this client */ - if (mods_lookup_irq(channel, dev, 0) == IRQ_NOT_FOUND) { - mods_error_printk( - "IRQ for dev %04x:%x:%02x.%x not hooked\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); - LOG_EXT(); - return -EINVAL; - } - - /* Delete device interrupt from the list */ - list_for_each_entry_safe(del, next, &pmp->irq_head[channel - 1], list) { - if (dev == del->dev) { - if (del->type != p->type) { - mods_error_printk("wrong IRQ type passed\n"); - LOG_EXT(); - return -EINVAL; - } - list_del(&del->list); - mods_debug_printk(DEBUG_ISR, - "%04x:%x:%02x.%x unregistered %s IRQ 0x%x\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function, - (del->type == MODS_IRQ_TYPE_MSI) - ? "MSI" : "INTx", - del->dev->irq); - mods_free_map(del); - break; - } - } + rv = mods_free_irqs(channel, dev); LOG_EXT(); - return OK; + return rv; } #endif @@ -888,11 +1143,12 @@ static int mods_unregister_cpu_irq(struct file *pfile, * ESCAPE CALL FUNCTIONS * *************************/ - -int esc_mods_register_irq_3(struct file *pfile, - struct MODS_REGISTER_IRQ_3 *p) +int esc_mods_register_irq_4(struct file *pfile, + struct MODS_REGISTER_IRQ_4 *p) { - if (p->irq_type == MODS_IRQ_TYPE_CPU) + u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); + + if (irq_type == MODS_IRQ_TYPE_CPU) return mods_register_cpu_irq(pfile, p); #ifdef CONFIG_PCI return mods_register_pci_irq(pfile, p); @@ -902,13 +1158,45 @@ int esc_mods_register_irq_3(struct file *pfile, #endif } +int esc_mods_register_irq_3(struct file *pfile, + struct MODS_REGISTER_IRQ_3 *p) +{ + struct MODS_REGISTER_IRQ_4 irq_data = { {0} }; + u32 ii = 0; + + irq_data.dev = p->dev; + irq_data.aperture_addr = p->aperture_addr; + irq_data.aperture_size = p->aperture_size; + irq_data.mask_info_cnt = p->mask_info_cnt; + for (ii = 0; ii < p->mask_info_cnt; ii++) { + irq_data.mask_info[ii].mask_type = p->mask_info[ii].mask_type; + irq_data.mask_info[ii].irq_pending_offset = + p->mask_info[ii].irq_pending_offset; + irq_data.mask_info[ii].irq_enabled_offset = + p->mask_info[ii].irq_enabled_offset; + irq_data.mask_info[ii].irq_enable_offset = + p->mask_info[ii].irq_enable_offset; + irq_data.mask_info[ii].irq_disable_offset = + p->mask_info[ii].irq_disable_offset; + irq_data.mask_info[ii].and_mask = + p->mask_info[ii].and_mask; + irq_data.mask_info[ii].or_mask = + p->mask_info[ii].or_mask; + } + irq_data.irq_count = 1; + irq_data.irq_flags = p->irq_type; + + return esc_mods_register_irq_4(pfile, &irq_data); +} + int esc_mods_register_irq_2(struct file *pfile, struct MODS_REGISTER_IRQ_2 *p) { - struct MODS_REGISTER_IRQ_3 irq_data = { {0} }; + struct MODS_REGISTER_IRQ_4 irq_data = { {0} }; irq_data.dev = p->dev; - irq_data.irq_type = p->type; + irq_data.irq_count = 1; + irq_data.irq_flags = p->type; if (p->type == MODS_IRQ_TYPE_CPU) return mods_register_cpu_irq(pfile, &irq_data); #ifdef CONFIG_PCI @@ -979,7 +1267,7 @@ int esc_mods_unregister_irq(struct file *pfile, return esc_mods_unregister_irq_2(pfile, ®ister_irq); } -int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) +int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) { unsigned char channel; struct irq_q_info *q = NULL; @@ -1019,16 +1307,18 @@ int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) p->irq_list[i].dev.function = PCI_FUNC(dev->devfn); } else { p->irq_list[i].dev.domain = 0; - p->irq_list[i].dev.bus = q->data[index].irq; + p->irq_list[i].dev.bus = 0; p->irq_list[i].dev.device = 0xFFU; p->irq_list[i].dev.function = 0xFFU; } + p->irq_list[i].irq_index = q->data[index].irq_index; p->irq_list[i].delay = cur_time - q->data[index].time; /* Print info about IRQ status returned */ if (dev) { mods_debug_printk(DEBUG_ISR_DETAILED, - "retrieved IRQ dev %04x:%x:%02x.%x, time=%uus, delay=%uus\n", + "retrieved IRQ index=%d dev %04x:%x:%02x.%x, time=%uus, delay=%uus\n", + p->irq_list[i].irq_index, (unsigned int)p->irq_list[i].dev.domain, (unsigned int)p->irq_list[i].dev.bus, (unsigned int)p->irq_list[i].dev.device, @@ -1055,13 +1345,30 @@ int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) return OK; } +int esc_mods_query_irq_2(struct file *pfile, struct MODS_QUERY_IRQ_2 *p) +{ + int retval, i; + struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; + + retval = esc_mods_query_irq_3(pfile, &query_irq); + if (retval) + return retval; + + for (i = 0; i < MODS_MAX_IRQS; i++) { + p->irq_list[i].dev = query_irq.irq_list[i].dev; + p->irq_list[i].delay = query_irq.irq_list[i].delay; + } + p->more = query_irq.more; + return OK; +} + int esc_mods_query_irq(struct file *pfile, struct MODS_QUERY_IRQ *p) { int retval, i; - struct MODS_QUERY_IRQ_2 query_irq = { { {0} } }; + struct MODS_QUERY_IRQ_3 query_irq = { { { { 0 } } } }; - retval = esc_mods_query_irq_2(pfile, &query_irq); + retval = esc_mods_query_irq_3(pfile, &query_irq); if (retval) return retval; @@ -1109,7 +1416,7 @@ int esc_mods_set_irq_multimask(struct file *pfile, mods_debug_printk( DEBUG_ISR, "set %s IRQ mask dev %04x:%x:%02x.%x mask t%d &0x%llx |0x%llx addr=0x%llx\n", - p->irq_type == MODS_IRQ_TYPE_INT ? "INT" : "MSI", + mods_irq_type_name(p->irq_type), (unsigned int)p->dev.domain, (unsigned int)p->dev.bus, (unsigned int)p->dev.device, @@ -1154,7 +1461,8 @@ int esc_mods_set_irq_multimask(struct file *pfile, return -EINVAL; #endif } else { - mods_error_printk("set_irq_multimask not supported for MSI!\n"); + mods_error_printk("set_irq_multimask not supported for %s!\n", + mods_irq_type_name(p->irq_type)); LOG_EXT(); return -EINVAL; } diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c index 7b18e279..7a5b39c6 100644 --- a/drivers/misc/mods/mods_krnl.c +++ b/drivers/misc/mods/mods_krnl.c @@ -19,13 +19,15 @@ #include "mods_internal.h" -#include +#include #include -#include #include +#include +#include +#include +#include #include #include -#include #ifdef MODS_HAS_CONSOLE_LOCK # include # include @@ -43,8 +45,12 @@ static unsigned int mods_krnl_poll(struct file *, poll_table *); static int mods_krnl_mmap(struct file *, struct vm_area_struct *); static long mods_krnl_ioctl(struct file *, unsigned int, unsigned long); +#ifdef MODS_HAS_SRIOV +static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs); +#endif + /* character driver entry points */ -const struct file_operations mods_fops = { +static const struct file_operations mods_fops = { .owner = THIS_MODULE, .open = mods_krnl_open, .release = mods_krnl_close, @@ -98,14 +104,20 @@ static const struct pci_device_id mods_pci_table[] = { static int mods_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { + mods_debug_printk(DEBUG_PCI, "probed vendor %x device %x devfn %x\n", + dev->vendor, dev->device, dev->devfn); + return 0; } struct pci_driver mods_pci_driver = { - .name = DEVICE_NAME, - .id_table = mods_pci_table, - .probe = mods_pci_probe, - .err_handler = &mods_pci_error_handlers, + .name = DEVICE_NAME, + .id_table = mods_pci_table, + .probe = mods_pci_probe, + .err_handler = &mods_pci_error_handlers, +#ifdef MODS_HAS_SRIOV + .sriov_configure = mods_pci_sriov_configure, +#endif }; #endif @@ -118,6 +130,105 @@ static int debug = -0x80000000; static int multi_instance = MODS_MULTI_INSTANCE_DEFAULT_VALUE; static u32 access_token = MODS_ACCESS_TOKEN_NONE; +#ifdef MODS_HAS_SRIOV +static int mods_pci_sriov_configure(struct pci_dev *dev, int numvfs) +{ + int rv = 0; + + LOG_ENT(); + + mods_debug_printk(DEBUG_PCI, + "(numvfs=%d, totalvfs=%d, dev->is_physfn=%d)\n", + numvfs, + pci_sriov_get_totalvfs(dev), + dev->is_physfn); + + if (numvfs > 0) { + rv = pci_enable_sriov(dev, numvfs); + if (rv) { + mods_error_printk( + "pci_enable_sriov failed error %d\n", rv); + return rv; + } + + rv = numvfs; + } else { + pci_disable_sriov(dev); + rv = 0; + } + + LOG_EXT(); + return rv; +} + +static int esc_mods_set_num_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) +{ + int rv = 0; + struct pci_dev *dev; + unsigned int devfn; + + LOG_ENT(); + + /* Get the PCI device structure for the specified device from kernel */ + devfn = PCI_DEVFN(p->dev.device, p->dev.function); + dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); + if (!dev) { + mods_error_printk( + "unknown dev %04x:%x:%02x.%x\n", + (unsigned int)p->dev.domain, + (unsigned int)p->dev.bus, + (unsigned int)p->dev.device, + (unsigned int)p->dev.function); + LOG_EXT(); + return -EINVAL; + } + + rv = mods_pci_sriov_configure(dev, p->numvfs); + + LOG_EXT(); + + return rv; +} + +static int esc_mods_set_total_vf(struct file *pfile, struct MODS_SET_NUM_VF *p) +{ + int rv = 0; + struct pci_dev *dev; + unsigned int devfn; + + LOG_ENT(); + + mods_debug_printk(DEBUG_PCI, + "pci_sriov_set_totalvfs(totalvfs=%d)\n", p->numvfs); + + /* Get the PCI device structure for the specified device from kernel */ + devfn = PCI_DEVFN(p->dev.device, p->dev.function); + dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); + if (!dev) { + mods_error_printk( + "unknown dev %04x:%x:%02x.%x\n", + (unsigned int)p->dev.domain, + (unsigned int)p->dev.bus, + (unsigned int)p->dev.device, + (unsigned int)p->dev.function); + LOG_EXT(); + return -EINVAL; + } + + rv = pci_sriov_set_totalvfs(dev, p->numvfs); + + if (rv) { + mods_error_printk( + "pci_sriov_set_totalvfs failed error %d\n", rv); + return rv; + } + + LOG_EXT(); + + return rv; +} +#endif + #if defined(CONFIG_PPC64) static int ppc_tce_bypass = MODS_PPC_TCE_BYPASS_ON; @@ -229,6 +340,10 @@ static int __init mods_init_module(void) mods_info_printk("driver loaded, version %x.%02x\n", (MODS_DRIVER_VERSION>>8), (MODS_DRIVER_VERSION&0xFF)); + + if (debug) + mods_info_printk("debug level 0x%x\n", debug); + LOG_EXT(); return OK; } @@ -301,7 +416,7 @@ static void mods_disable_all_devices(struct mods_file_private_data *priv) while (priv->enabled_devices != 0) { struct en_dev_entry *old = priv->enabled_devices; - pci_disable_device(old->dev); + mods_disable_device(old->dev); priv->enabled_devices = old->next; kfree(old); } @@ -320,7 +435,7 @@ static int mods_register_mapping( struct MODS_MEM_INFO *p_mem_info, u64 dma_addr, u64 virtual_address, - u32 mapping_length) + u64 mapping_length) { struct SYS_MAP_MEMORY *p_map_mem; MODS_PRIV private_data = fp->private_data; @@ -342,7 +457,7 @@ static int mods_register_mapping( list_add(&p_map_mem->list, private_data->mods_mapping_list); mods_debug_printk(DEBUG_MEM_DETAILED, - "map alloc %p as %p: phys 0x%llx, virt 0x%llx, size 0x%x\n", + "map alloc %p as %p: phys 0x%llx, virt 0x%llx, size 0x%llx\n", p_mem_info, p_map_mem, dma_addr, virtual_address, mapping_length); LOG_EXT(); @@ -463,7 +578,7 @@ static const char *mods_get_prot_str_for_range(struct file *fp, static pci_ers_result_t mods_pci_error_detected(struct pci_dev *dev, enum pci_channel_state state) { - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci_error_detected %04x:%x:%02x.%x\n", pci_domain_nr(dev->bus), dev->bus->number, @@ -475,7 +590,7 @@ static pci_ers_result_t mods_pci_error_detected(struct pci_dev *dev, static pci_ers_result_t mods_pci_mmio_enabled(struct pci_dev *dev) { - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci_mmio_enabled %04x:%x:%02x.%x\n", pci_domain_nr(dev->bus), dev->bus->number, @@ -487,7 +602,7 @@ static pci_ers_result_t mods_pci_mmio_enabled(struct pci_dev *dev) static void mods_pci_resume(struct pci_dev *dev) { - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci_resume %04x:%x:%02x.%x\n", pci_domain_nr(dev->bus), dev->bus->number, @@ -913,10 +1028,10 @@ static int mods_krnl_map_inner(struct file *fp, struct vm_area_struct *vma) /* device memory */ mods_debug_printk(DEBUG_MEM, - "map dev: phys 0x%llx, virt 0x%lx, size 0x%x, %s\n", + "map dev: phys 0x%llx, virt 0x%lx, size 0x%lx, %s\n", req_pa, (unsigned long)vma->vm_start, - (unsigned int)MODS_VMA_SIZE(vma), + (unsigned long)MODS_VMA_SIZE(vma), mods_get_prot_str_for_range(fp, req_pa, MODS_VMA_SIZE(vma))); @@ -1201,6 +1316,87 @@ static int esc_mods_verify_access_token(struct file *pfile, return ret; } +struct mods_sysfs_work { + struct work_struct work; + struct MODS_SYSFS_NODE *pdata; + int ret; +}; + +#ifdef MODS_OLD_INIT_WORK +static void sysfs_write_task(void *w) +#else +static void sysfs_write_task(struct work_struct *w) +#endif +{ + struct mods_sysfs_work *task = container_of(w, + struct mods_sysfs_work, + work); + struct file *f; + mm_segment_t old_fs; + + LOG_ENT(); + + task->ret = -EINVAL; + + old_fs = get_fs(); + set_fs(KERNEL_DS); + + f = filp_open(task->pdata->path, O_WRONLY, 0); + if (IS_ERR(f)) + task->ret = PTR_ERR(f); + else { + f->f_pos = 0; + if (task->pdata->size <= MODS_MAX_SYSFS_FILE_SIZE) + task->ret = f->f_op->write(f, + task->pdata->contents, + task->pdata->size, + &f->f_pos); + filp_close(f, NULL); + } + + set_fs(old_fs); + + LOG_EXT(); +} + +static int esc_mods_write_sysfs_node(struct file *pfile, + struct MODS_SYSFS_NODE *pdata) +{ + int ret = -EINVAL; + struct mods_sysfs_work task; + struct workqueue_struct *wq; + + LOG_ENT(); + + memmove(&pdata->path[5], pdata->path, MODS_MAX_SYSFS_PATH_LEN); + memcpy(pdata->path, "/sys/", 5); + pdata->path[MODS_MAX_SYSFS_PATH_BUF_SIZE - 1] = 0; + + task.pdata = pdata; + + wq = create_singlethread_workqueue("mods_sysfs_write"); + if (!wq) { + LOG_EXT(); + return ret; + } + +#ifdef MODS_OLD_INIT_WORK + INIT_WORK(&task.work, sysfs_write_task, &task); +#else + INIT_WORK(&task.work, sysfs_write_task); +#endif + queue_work(wq, &task.work); + flush_workqueue(wq); + destroy_workqueue(wq); + + ret = task.ret; + if (ret > 0) + ret = OK; + + LOG_EXT(); + return ret; +} + /************** * IO control * **************/ @@ -1446,6 +1642,12 @@ static long mods_krnl_ioctl(struct file *fp, MODS_GET_PHYSICAL_ADDRESS); break; + case MODS_ESC_GET_PHYSICAL_ADDRESS_2: + MODS_IOCTL(MODS_ESC_GET_PHYSICAL_ADDRESS_2, + esc_mods_get_phys_addr_2, + MODS_GET_PHYSICAL_ADDRESS_3); + break; + case MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS: MODS_IOCTL(MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS, esc_mods_get_mapped_phys_addr, @@ -1458,6 +1660,12 @@ static long mods_krnl_ioctl(struct file *fp, MODS_GET_PHYSICAL_ADDRESS_2); break; + case MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3: + MODS_IOCTL(MODS_ESC_GET_MAPPED_PHYSICAL_ADDRESS_3, + esc_mods_get_mapped_phys_addr_3, + MODS_GET_PHYSICAL_ADDRESS_3); + break; + case MODS_ESC_SET_MEMORY_TYPE: MODS_IOCTL_NORETVAL(MODS_ESC_SET_MEMORY_TYPE, esc_mods_set_mem_type, @@ -1891,8 +2099,38 @@ static long mods_krnl_ioctl(struct file *fp, MODS_ACCESS_TOKEN); break; + case MODS_ESC_WRITE_SYSFS_NODE: + MODS_IOCTL_NORETVAL(MODS_ESC_WRITE_SYSFS_NODE, + esc_mods_write_sysfs_node, + MODS_SYSFS_NODE); + break; + + case MODS_ESC_REGISTER_IRQ_4: + MODS_IOCTL_NORETVAL(MODS_ESC_REGISTER_IRQ_4, + esc_mods_register_irq_4, MODS_REGISTER_IRQ_4); + break; + + case MODS_ESC_QUERY_IRQ_3: + MODS_IOCTL(MODS_ESC_QUERY_IRQ_3, + esc_mods_query_irq_3, MODS_QUERY_IRQ_3); + break; + +#ifdef MODS_HAS_SRIOV + case MODS_ESC_SET_NUM_VF: + MODS_IOCTL_NORETVAL(MODS_ESC_SET_NUM_VF, + esc_mods_set_num_vf, MODS_SET_NUM_VF); + break; + + case MODS_ESC_SET_TOTAL_VF: + MODS_IOCTL_NORETVAL(MODS_ESC_SET_TOTAL_VF, + esc_mods_set_total_vf, MODS_SET_NUM_VF); + break; +#endif + default: - mods_error_printk("unrecognized ioctl (0x%x)\n", cmd); + mods_error_printk("unrecognized ioctl (0x%x) dir(0x%x) type (0x%x) nr (0x%x) size (0x%x)\n", + cmd, _IOC_DIR(cmd), _IOC_TYPE(cmd), + _IOC_NR(cmd), _IOC_SIZE(cmd)); ret = -EINVAL; break; } diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c index d04a09db..b3f606b1 100644 --- a/drivers/misc/mods/mods_mem.c +++ b/drivers/misc/mods/mods_mem.c @@ -1,7 +1,7 @@ /* * mods_mem.c - This file is part of NVIDIA MODS kernel driver. * - * Copyright (c) 2008-2017, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2008-2018, NVIDIA CORPORATION. All rights reserved. * * NVIDIA MODS kernel driver is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License, @@ -559,7 +559,7 @@ int mods_unregister_all_alloc(struct file *fp) */ int mods_get_alloc_offset(struct MODS_MEM_INFO *p_mem_info, u64 dma_addr, - u32 *ret_offs) + u64 *ret_offs) { u32 i; u64 offset = 0; @@ -567,7 +567,7 @@ int mods_get_alloc_offset(struct MODS_MEM_INFO *p_mem_info, for (i = 0; i < p_mem_info->max_chunks; i++) { struct MODS_PHYS_CHUNK *pt = &p_mem_info->pages[i]; u64 addr = pt->dma_addr; - u32 size = PAGE_SIZE << pt->order; + u64 size = PAGE_SIZE << pt->order; if (!pt->allocated) break; @@ -591,7 +591,7 @@ struct MODS_MEM_INFO *mods_find_alloc(struct file *fp, u64 phys_addr) struct list_head *plist_head = private_data->mods_alloc_list; struct list_head *plist_iter; struct MODS_MEM_INFO *p_mem_info; - u32 offset; + u64 offset; list_for_each(plist_iter, plist_head) { p_mem_info = list_entry(plist_iter, @@ -631,12 +631,12 @@ static u32 mods_estimate_max_chunks(u32 num_pages) static struct MODS_PHYS_CHUNK *mods_find_phys_chunk( struct MODS_MEM_INFO *p_mem_info, - u32 offset, - u32 *chunk_offset) + u64 offset, + u64 *chunk_offset) { struct MODS_PHYS_CHUNK *pt = NULL; - u32 pages_left; - u32 page_offs; + u64 pages_left; + u64 page_offs; u32 i; if (!p_mem_info) @@ -917,10 +917,31 @@ int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) } int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) +{ + int retval; + struct MODS_GET_PHYSICAL_ADDRESS_3 get_phys_addr_3; + + LOG_ENT(); + + memset(&get_phys_addr_3, 0, + sizeof(struct MODS_GET_PHYSICAL_ADDRESS_3)); + get_phys_addr_3.memory_handle = p->memory_handle; + get_phys_addr_3.offset = p->offset; + + retval = esc_mods_get_phys_addr_2(fp, &get_phys_addr_3); + if (!retval) + p->physical_address = get_phys_addr_3.physical_address; + + LOG_EXT(); + return retval; +} + +int esc_mods_get_phys_addr_2(struct file *fp, + struct MODS_GET_PHYSICAL_ADDRESS_3 *p) { struct MODS_MEM_INFO *p_mem_info; struct MODS_PHYS_CHUNK *pt = NULL; - u32 chunk_offset; + u64 chunk_offset; LOG_ENT(); @@ -935,7 +956,7 @@ int esc_mods_get_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) p->physical_address = pt->dma_addr + chunk_offset; mods_debug_printk(DEBUG_MEM_DETAILED, - "get phys: %p+0x%x -> 0x%llx\n", + "get phys: %p+0x%llx -> 0x%llx\n", p_mem_info, p->offset, p->physical_address); LOG_EXT(); return 0; @@ -945,30 +966,30 @@ int esc_mods_get_mapped_phys_addr(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS *p) { int retval; - struct MODS_GET_PHYSICAL_ADDRESS_2 get_mapped_phys_addr_2; + struct MODS_GET_PHYSICAL_ADDRESS_3 get_mapped_phys_addr_3; struct MODS_MEM_INFO *p_mem_info; LOG_ENT(); - memset(&get_mapped_phys_addr_2, 0, - sizeof(struct MODS_GET_PHYSICAL_ADDRESS_2)); - get_mapped_phys_addr_2.memory_handle = p->memory_handle; - get_mapped_phys_addr_2.offset = p->offset; + memset(&get_mapped_phys_addr_3, 0, + sizeof(struct MODS_GET_PHYSICAL_ADDRESS_3)); + get_mapped_phys_addr_3.memory_handle = p->memory_handle; + get_mapped_phys_addr_3.offset = p->offset; p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; if (p_mem_info->dev) { - get_mapped_phys_addr_2.pci_device.domain = + get_mapped_phys_addr_3.pci_device.domain = pci_domain_nr(p_mem_info->dev->bus); - get_mapped_phys_addr_2.pci_device.bus = + get_mapped_phys_addr_3.pci_device.bus = p_mem_info->dev->bus->number; - get_mapped_phys_addr_2.pci_device.device = + get_mapped_phys_addr_3.pci_device.device = PCI_SLOT(p_mem_info->dev->devfn); - get_mapped_phys_addr_2.pci_device.function = + get_mapped_phys_addr_3.pci_device.function = PCI_FUNC(p_mem_info->dev->devfn); } - retval = esc_mods_get_mapped_phys_addr_2(fp, &get_mapped_phys_addr_2); + retval = esc_mods_get_mapped_phys_addr_3(fp, &get_mapped_phys_addr_3); if (!retval) - p->physical_address = get_mapped_phys_addr_2.physical_address; + p->physical_address = get_mapped_phys_addr_3.physical_address; LOG_EXT(); return retval; @@ -976,12 +997,34 @@ int esc_mods_get_mapped_phys_addr(struct file *fp, int esc_mods_get_mapped_phys_addr_2(struct file *fp, struct MODS_GET_PHYSICAL_ADDRESS_2 *p) +{ + int retval; + struct MODS_GET_PHYSICAL_ADDRESS_3 get_mapped_phys_addr_3; + + LOG_ENT(); + + memset(&get_mapped_phys_addr_3, 0, + sizeof(struct MODS_GET_PHYSICAL_ADDRESS_3)); + get_mapped_phys_addr_3.memory_handle = p->memory_handle; + get_mapped_phys_addr_3.offset = p->offset; + get_mapped_phys_addr_3.pci_device = p->pci_device; + + retval = esc_mods_get_mapped_phys_addr_3(fp, &get_mapped_phys_addr_3); + if (!retval) + p->physical_address = get_mapped_phys_addr_3.physical_address; + + LOG_EXT(); + return retval; +} + +int esc_mods_get_mapped_phys_addr_3(struct file *fp, + struct MODS_GET_PHYSICAL_ADDRESS_3 *p) { struct pci_dev *dev = NULL; struct MODS_MEM_INFO *p_mem_info; struct MODS_PHYS_CHUNK *pt; struct MODS_MAP_CHUNK *pm; - u32 chunk_offset; + u64 chunk_offset; p_mem_info = (struct MODS_MEM_INFO *)(size_t)p->memory_handle; pt = mods_find_phys_chunk(p_mem_info, p->offset, &chunk_offset); @@ -1002,7 +1045,7 @@ int esc_mods_get_mapped_phys_addr_2(struct file *fp, if (!dev) { mods_debug_printk(DEBUG_MEM_DETAILED, - "get mapped phys: %p+0x%x -> 0x%llx\n", + "get mapped phys: %p+0x%llx -> 0x%llx\n", p_mem_info, p->offset, p->physical_address); p->physical_address = pt->dma_addr + chunk_offset; LOG_EXT(); @@ -1018,7 +1061,7 @@ int esc_mods_get_mapped_phys_addr_2(struct file *fp, p->physical_address = pm->map_addr + chunk_offset; mods_debug_printk(DEBUG_MEM_DETAILED, - "get mapped phys: %p+0x%x -> 0x%llx\n", + "get mapped phys: %p+0x%llx -> 0x%llx\n", p_mem_info, p->offset, p->physical_address); LOG_EXT(); return 0; @@ -1044,7 +1087,7 @@ int esc_mods_virtual_to_phys(struct file *fp, list_for_each(iter, head) { struct SYS_MAP_MEMORY *p_map_mem; u64 begin, end; - u32 phys_offs; + u64 phys_offs; p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list); @@ -1053,7 +1096,7 @@ int esc_mods_virtual_to_phys(struct file *fp, if (p->virtual_address >= begin && p->virtual_address < end) { - u32 virt_offs = p->virtual_address - begin; + u64 virt_offs = p->virtual_address - begin; int ret; /* device memory mapping */ @@ -1110,8 +1153,8 @@ int esc_mods_phys_to_virtual(struct file *fp, MODS_PRIV private_data = fp->private_data; struct list_head *head; struct list_head *iter; - u32 offset; - u32 map_offset; + u64 offset; + u64 map_offset; LOG_ENT(); diff --git a/drivers/misc/mods/mods_pci.c b/drivers/misc/mods/mods_pci.c index 6a9b5a29..e5cdfb05 100644 --- a/drivers/misc/mods/mods_pci.c +++ b/drivers/misc/mods/mods_pci.c @@ -38,7 +38,7 @@ static int mods_free_pci_res_map(struct file *fp, struct list_head *head; struct list_head *iter; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "free pci resource map %p\n", p_del_map); @@ -60,7 +60,7 @@ static int mods_free_pci_res_map(struct file *fp, p_res_map->va, p_res_map->page_count * PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "unmapped pci resource at 0x%llx from %u:%u:%u.%u\n", p_res_map->va, pci_domain_nr(p_res_map->dev->bus), @@ -114,7 +114,7 @@ static int mods_find_pci_dev(struct file *pfile, struct pci_dev *dev = NULL; int index = -1; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "find pci dev %04x:%04x, index %d\n", (int) p->vendor_id, (int) p->device_id, @@ -184,7 +184,7 @@ static int mods_find_pci_class_code(struct file *pfile, struct pci_dev *dev = NULL; int index = -1; - mods_debug_printk(DEBUG_PCICFG, "find pci class code %04x, index %d\n", + mods_debug_printk(DEBUG_PCI, "find pci class code %04x, index %d\n", (int) p->class_code, (int) p->index); do { @@ -258,7 +258,7 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, if (dev == NULL) return -EINVAL; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci get bar info %04x:%x:%02x:%x, bar index %d\n", (int) p->pci_device.domain, (int) p->pci_device.bus, (int) p->pci_device.device, @@ -320,7 +320,7 @@ int esc_mods_pci_get_irq_2(struct file *pfile, if (dev == NULL) return -EINVAL; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci get irq %04x:%x:%02x:%x\n", (int) p->pci_device.domain, (int) p->pci_device.bus, (int) p->pci_device.device, @@ -361,7 +361,7 @@ int esc_mods_pci_read_2(struct file *pfile, struct MODS_PCI_READ_2 *p) if (dev == NULL) return -EINVAL; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci read %04x:%x:%02x.%x, addr 0x%04x, size %d\n", (int) p->pci_device.domain, (int) p->pci_device.bus, (int) p->pci_device.device, @@ -371,17 +371,17 @@ int esc_mods_pci_read_2(struct file *pfile, struct MODS_PCI_READ_2 *p) p->data = 0; switch (p->data_size) { case 1: { - u8 value; + u8 value; - pci_read_config_byte(dev, p->address, &value); - p->data = value; + pci_read_config_byte(dev, p->address, &value); + p->data = value; } break; case 2: { - u16 value; + u16 value; - pci_read_config_word(dev, p->address, &value); - p->data = value; + pci_read_config_word(dev, p->address, &value); + p->data = value; } break; case 4: @@ -418,7 +418,7 @@ int esc_mods_pci_write_2(struct file *pfile, struct MODS_PCI_WRITE_2 *p) struct pci_dev *dev; unsigned int devfn; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci write %04x:%x:%02x.%x, addr 0x%04x, size %d, data 0x%x\n", (int) p->pci_device.domain, (int) p->pci_device.bus, (int) p->pci_device.device, @@ -498,7 +498,7 @@ int esc_mods_pci_hot_reset(struct file *pfile, unsigned int devfn; int retval; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "pci_hot_reset %04x:%x:%02x.%x\n", (int) p->pci_device.domain, (int) p->pci_device.bus, @@ -765,7 +765,7 @@ int esc_mods_pci_map_resource(struct file *fp, p->va = p_res_map->va; - mods_debug_printk(DEBUG_PCICFG, + mods_debug_printk(DEBUG_PCI, "mapped pci resource %u from %u:%u:%u.%u to %u:%u:%u.%u at 0x%llx\n", p->resource_index, p->remote_pci_device.domain, @@ -845,9 +845,10 @@ int esc_mods_get_iommu_state(struct file *pfile, struct MODS_GET_IOMMU_STATE *state) { #if !defined(CONFIG_SWIOTLB) - /* SW IOTLB turned off in the kernel, HW IOMMU active */ + /* SWIOTLB turned off in the kernel, HW IOMMU active */ state->state = 1; -#elif defined(MODS_HAS_DMA_OPS) +#elif defined(MODS_HAS_DMA_OPS) && \ + (defined(MODS_HAS_NONCOH_DMA_OPS) || defined(MODS_HAS_MAP_SG_ATTRS)) unsigned int devfn = PCI_DEVFN(state->pci_device.device, state->pci_device.function); @@ -858,13 +859,16 @@ int esc_mods_get_iommu_state(struct file *pfile, const struct dma_map_ops *ops = get_dma_ops(&dev->dev); #if defined(MODS_HAS_NONCOH_DMA_OPS) - state->state = ops->map_sg != &noncoherent_swiotlb_dma_ops && - ops->map_sg != &coherent_swiotlb_dma_ops; + state->state = ops != &noncoherent_swiotlb_dma_ops && + ops != &coherent_swiotlb_dma_ops; #else state->state = ops->map_sg != swiotlb_map_sg_attrs; #endif +#elif defined(CONFIG_PPC64) || defined(CONFIG_ARM64) + /* Old/new kernels, no way to detect, default to HW IOMMU active */ + state->state = 1; #else - /* Old kernels, only x86 support, assume no IOMMU */ + /* Old/new kernels, no way to detect, on x86 default to no IOMMU */ state->state = 0; #endif return OK;