diff --git a/drivers/misc/mods/mods.h b/drivers/misc/mods/mods.h index 69c12cda..79cb5183 100644 --- a/drivers/misc/mods/mods.h +++ b/drivers/misc/mods/mods.h @@ -24,7 +24,7 @@ /* Driver version */ #define MODS_DRIVER_VERSION_MAJOR 3 -#define MODS_DRIVER_VERSION_MINOR 86 +#define MODS_DRIVER_VERSION_MINOR 87 #define MODS_DRIVER_VERSION ((MODS_DRIVER_VERSION_MAJOR << 8) | \ ((MODS_DRIVER_VERSION_MINOR/10) << 4) | \ (MODS_DRIVER_VERSION_MINOR%10)) @@ -161,6 +161,17 @@ struct MODS_DMA_MAP_MEMORY { struct mods_pci_dev_2 pci_device; }; +/* MODS_ESC_PCI_SET_DMA_MASK */ +struct MODS_PCI_DMA_MASK { + /* IN */ + struct mods_pci_dev_2 pci_device; + __u32 num_bits; +}; + +#define MODS_SWIOTLB_DISABLED 0 +#define MODS_SWIOTLB_ACTIVE 1 +#define MODS_SWIOTLB_INDETERMINATE 2 + /* MODS_ESC_GET_IOMMU_STATE */ struct MODS_GET_IOMMU_STATE { /* IN */ @@ -507,6 +518,7 @@ struct mods_mask_info { __u64 and_mask; /*and mask for clearing bits in this register */ __u64 or_mask; /*or value for setting bit in this register */ }; + struct MODS_SET_IRQ_MULTIMASK { /* IN */ __u64 aperture_addr; /* physical address of aperture */ @@ -518,7 +530,6 @@ struct MODS_SET_IRQ_MULTIMASK { __u8 irq_type; /* irq type */ }; - /* MODS_ESC_SET_IRQ_MASK_2 */ struct MODS_SET_IRQ_MASK_2 { /* IN */ @@ -785,6 +796,37 @@ struct MODS_TEGRA_DC_CONFIG_POSSIBLE { __u8 possible; }; + +#define MODS_TEGRA_DC_SETUP_SD_LUT_SIZE 9 +#define MODS_TEGRA_DC_SETUP_BLTF_SIZE 16 +/* MODS_ESC_TEGRA_DC_SETUP_SD */ +struct MODS_TEGRA_DC_SETUP_SD { + /* IN */ + __u8 head; + __u8 enable; + + __u8 use_vid_luma; + __u8 csc_r; + __u8 csc_g; + __u8 csc_b; + __u8 aggressiveness; + __u8 bin_width_log2; + + __u32 lut[MODS_TEGRA_DC_SETUP_SD_LUT_SIZE]; + __u32 bltf[MODS_TEGRA_DC_SETUP_BLTF_SIZE]; + + __u32 klimit; + __u32 soft_clipping_threshold; + __u32 smooth_k_inc; + __u8 k_init_bias; + + + __u32 win_x; + __u32 win_y; + __u32 win_w; + __u32 win_h; +}; + /* MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS */ struct MODS_DMABUF_GET_PHYSICAL_ADDRESS { /* IN */ @@ -1132,6 +1174,8 @@ struct MODS_SET_NUM_VF { #define MODS_ESC_TEGRA_DC_CONFIG_POSSIBLE \ _IOWR(MODS_IOC_MAGIC, 47, \ struct MODS_TEGRA_DC_CONFIG_POSSIBLE) +#define MODS_ESC_TEGRA_DC_SETUP_SD \ + _IOW(MODS_IOC_MAGIC, 48, struct MODS_TEGRA_DC_SETUP_SD) #define MODS_ESC_DMABUF_GET_PHYSICAL_ADDRESS \ _IOWR(MODS_IOC_MAGIC, 49, \ struct MODS_DMABUF_GET_PHYSICAL_ADDRESS) @@ -1290,5 +1334,9 @@ struct MODS_SET_NUM_VF { _IOW(MODS_IOC_MAGIC, 116, struct MODS_SET_NUM_VF) #define MODS_ESC_SET_TOTAL_VF \ _IOW(MODS_IOC_MAGIC, 117, struct MODS_SET_NUM_VF) +#define MODS_ESC_PCI_SET_DMA_MASK \ + _IOW(MODS_IOC_MAGIC, 118, struct MODS_PCI_DMA_MASK) +#define MODS_ESC_GET_IOMMU_STATE_2 \ + _IOWR(MODS_IOC_MAGIC, 119, struct MODS_GET_IOMMU_STATE) #endif /* _MODS_H_ */ diff --git a/drivers/misc/mods/mods_internal.h b/drivers/misc/mods/mods_internal.h index d7f5b7f4..e413d6f4 100644 --- a/drivers/misc/mods/mods_internal.h +++ b/drivers/misc/mods/mods_internal.h @@ -55,10 +55,10 @@ struct en_dev_entry { struct pci_dev *dev; struct en_dev_entry *next; - __u32 irqs_allocated; - __u32 irq_flags; - __u32 nvecs; struct msix_entry *msix_entries; + u32 irq_flags; + u32 nvecs; + u8 client_id; }; struct mem_type { @@ -67,26 +67,41 @@ struct mem_type { u32 type; }; -/* file private data */ -struct mods_file_private_data { - struct list_head *mods_alloc_list; - struct list_head *mods_mapping_list; - struct list_head *mods_pci_res_map_list; +struct irq_q_data { + u32 time; + struct pci_dev *dev; + u32 irq; + u32 irq_index; +}; + +struct irq_q_info { + struct irq_q_data data[MODS_MAX_IRQS]; + u32 head; + u32 tail; +}; + +/* The driver can be opened simultaneously multiple times, from the same or from + * different processes. This structure tracks data specific to each open fd. + */ +struct mods_client { + struct list_head irq_list; + struct list_head mem_alloc_list; + struct list_head mem_map_list; #if defined(CONFIG_PPC64) - struct list_head *mods_ppc_tce_bypass_list; - struct list_head *mods_nvlink_sysmem_trained_list; + struct list_head ppc_tce_bypass_list; + struct list_head nvlink_sysmem_trained_list; #endif wait_queue_head_t interrupt_event; + struct irq_q_info irq_queue; + spinlock_t irq_lock; struct en_dev_entry *enabled_devices; - int mods_id; struct mem_type mem_type; struct mutex mtx; int mods_fb_suspended[FB_MAX]; u32 access_token; + u8 client_id; }; -#define MODS_PRIV struct mods_file_private_data * - /* VM private data */ struct mods_vm_private_data { struct file *fp; @@ -133,7 +148,7 @@ struct MODS_MEM_INFO { * memory was allocated on */ - struct list_head *dma_map_list; + struct list_head dma_map_list; struct list_head list; @@ -191,7 +206,7 @@ struct NVL_TRAINED { #define IRQ_MAX (256+PCI_IRQ_MAX) #define PCI_IRQ_MAX 15 -#define MODS_CHANNEL_MAX 32 +#define MODS_MAX_CLIENTS 32 #define IRQ_VAL_POISON 0xfafbfcfdU @@ -230,19 +245,6 @@ struct NVL_TRAINED { #define mods_warning_printk(fmt, args...)\ pr_info("mods warning: " fmt, ##args) -struct irq_q_data { - u32 time; - struct pci_dev *dev; - u32 irq; - u32 irq_index; -}; - -struct irq_q_info { - struct irq_q_data data[MODS_MAX_IRQS]; - u32 head; - u32 tail; -}; - struct irq_mask_info { u32 *dev_irq_mask_reg; /*IRQ mask register, read-only reg*/ u32 *dev_irq_state; /* IRQ status register*/ @@ -257,7 +259,7 @@ struct dev_irq_map { u32 apic_irq; u32 entry; u8 type; - u8 channel; + u8 client_id; u8 mask_info_cnt; struct irq_mask_info mask_info[MODS_IRQ_MAX_MASKS]; struct pci_dev *dev; @@ -265,16 +267,14 @@ struct dev_irq_map { }; struct mods_priv { - /* map info from pci irq to apic irq */ - struct list_head irq_head[MODS_CHANNEL_MAX]; + /* Bitmap for each allocated client id. */ + unsigned long client_flags; - /* bits map for each allocated id. Each mods has an id. */ - /* the design is to take into account multi mods. */ - unsigned long channel_flags; + /* Client structures */ + struct mods_client clients[MODS_MAX_CLIENTS]; - /* fifo loop queue */ - struct irq_q_info rec_info[MODS_CHANNEL_MAX]; - spinlock_t lock; + /* Mutex for guarding interrupt logic and PCI device enablement */ + struct mutex mtx; }; #ifndef MODS_HAS_SET_MEMORY @@ -362,9 +362,15 @@ struct mods_priv { #define MODS_ACPI_HANDLE(dev) DEVICE_ACPI_HANDLE(dev) #endif -/* FILE */ -#define MODS_GET_FILE_PRIVATE_ID(fp) (((struct mods_file_private_data *)(fp) \ - ->private_data)->mods_id) +static inline u8 get_client_id(struct file *fp) +{ + return ((struct mods_client *)(fp->private_data))->client_id; +} + +static inline int is_client_id_valid(u8 client_id) +{ + return client_id > 0 && client_id <= MODS_MAX_CLIENTS; +} /* ************************************************************************* */ /* ** MODULE WIDE FUNCTIONS */ @@ -373,11 +379,11 @@ struct mods_priv { /* irq */ void mods_init_irq(void); void mods_cleanup_irq(void); -unsigned char mods_alloc_channel(void); -void mods_free_channel(unsigned char channel); -void mods_irq_dev_clr_pri(unsigned char id); -void mods_irq_dev_set_pri(unsigned char id, void *pri); -int mods_irq_event_check(unsigned char channel); +struct mutex *mods_get_irq_mutex(void); +struct mods_client *mods_alloc_client(void); +void mods_free_client_interrupts(struct mods_client *client); +void mods_free_client(u8 client_id); +int mods_irq_event_check(u8 client_id); /* mem */ const char *mods_get_prot_str(u32 mem_type); @@ -392,13 +398,9 @@ int mods_unregister_all_nvlink_sysmem_trained(struct file *fp); #endif #ifdef CONFIG_PCI -int mods_enable_device(struct mods_file_private_data *priv, - struct pci_dev *pdev); +struct en_dev_entry *mods_enable_device(struct mods_client *client, + struct pci_dev *dev); void mods_disable_device(struct pci_dev *pdev); -int mods_unregister_all_pci_res_mappings(struct file *fp); -#define MODS_UNREGISTER_PCI_MAP(fp) mods_unregister_all_pci_res_mappings(fp) -#else -#define MODS_UNREGISTER_PCI_MAP(fp) 0 #endif /* clock */ @@ -489,12 +491,12 @@ int esc_mods_device_numa_info(struct file *fp, struct MODS_DEVICE_NUMA_INFO *p); int esc_mods_device_numa_info_2(struct file *fp, struct MODS_DEVICE_NUMA_INFO_2 *p); -int esc_mods_pci_map_resource(struct file *fp, - struct MODS_PCI_MAP_RESOURCE *p); -int esc_mods_pci_unmap_resource(struct file *fp, - struct MODS_PCI_UNMAP_RESOURCE *p); int esc_mods_get_iommu_state(struct file *pfile, struct MODS_GET_IOMMU_STATE *state); +int esc_mods_get_iommu_state_2(struct file *pfile, + struct MODS_GET_IOMMU_STATE *state); +int esc_mods_pci_set_dma_mask(struct file *pfile, + struct MODS_PCI_DMA_MASK *dma_mask); #endif /* irq */ #if defined(MODS_TEGRA) && defined(CONFIG_OF_IRQ) && defined(CONFIG_OF) @@ -510,11 +512,6 @@ int esc_mods_unregister_irq_2(struct file *fp, struct MODS_REGISTER_IRQ_2 *p); int esc_mods_query_irq(struct file *fp, struct MODS_QUERY_IRQ *p); int esc_mods_query_irq_2(struct file *fp, struct MODS_QUERY_IRQ_2 *p); -int esc_mods_set_irq_mask(struct file *fp, struct MODS_SET_IRQ_MASK *p); -int esc_mods_set_irq_mask_2(struct file *fp, - struct MODS_SET_IRQ_MASK_2 *p); -int esc_mods_set_irq_multimask(struct file *fp, - struct MODS_SET_IRQ_MULTIMASK *p); int esc_mods_irq_handled(struct file *fp, struct MODS_REGISTER_IRQ *p); int esc_mods_irq_handled_2(struct file *fp, struct MODS_REGISTER_IRQ_2 *p); diff --git a/drivers/misc/mods/mods_irq.c b/drivers/misc/mods/mods_irq.c index 9a5d72f1..c52ac72e 100644 --- a/drivers/misc/mods/mods_irq.c +++ b/drivers/misc/mods/mods_irq.c @@ -44,79 +44,73 @@ #define TOP_TKE_TKEIE_WDT_MASK(i) (1 << (16 + 4 * (i))) #define TOP_TKE_TKEIE(i) (0x100 + 4 * (i)) -struct nv_device { - char name[20]; - struct mods_priv *isr_pri; - void *pri[MODS_CHANNEL_MAX]; -}; - /********************* * PRIVATE FUNCTIONS * *********************/ static struct mods_priv mp; -static struct nv_device nv_dev = { "nvidia mods", &mp, {0} }; -static struct mods_priv *get_all_data(void) +struct mutex *mods_get_irq_mutex(void) { - return ∓ -} - -static struct nv_device *get_dev(void) -{ - return &nv_dev; + return &mp.mtx; } #ifdef CONFIG_PCI -int mods_enable_device(struct mods_file_private_data *priv, - struct pci_dev *pdev) +struct en_dev_entry *mods_enable_device(struct mods_client *client, + struct pci_dev *dev) { int ret = -1; - struct en_dev_entry *entry = priv->enabled_devices; + struct en_dev_entry *dpriv = client->enabled_devices; - mods_debug_printk(DEBUG_ISR_DETAILED, "vendor %x device %x device %x function %x\n", - pdev->vendor, pdev->device, - (unsigned int)PCI_SLOT(pdev->devfn), - (unsigned int)PCI_FUNC(pdev->devfn)); + BUG_ON(!mutex_is_locked(&mp.mtx)); - while (entry != 0) { - if (entry->dev == pdev) - return 0; - entry = entry->next; + dpriv = pci_get_drvdata(dev); + + if (dpriv) { + if (dpriv->client_id == client->client_id) + return dpriv; + + mods_error_printk("invalid client %u for device %04x:%x:%02x.%x\n", + (unsigned int)client->client_id, + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + return 0; } - ret = pci_enable_device(pdev); - if (ret == 0) { - entry = kzalloc(sizeof(*entry), GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!entry)) - return 0; - entry->dev = pdev; - entry->next = priv->enabled_devices; - priv->enabled_devices = entry; - pci_set_drvdata(pdev, entry); - mods_debug_printk(DEBUG_ISR_DETAILED, - "pci_set_drvdata for vendor %x device %x device %x function %x\n", - pdev->vendor, pdev->device, - (unsigned int)PCI_SLOT(pdev->devfn), - (unsigned int)PCI_FUNC(pdev->devfn)); - } else { - mods_debug_printk(DEBUG_ISR_DETAILED, - "pci_set_drvdata failed for vendor %x device %x device %x function %x ret=%d\n", - pdev->vendor, pdev->device, - (unsigned int)PCI_SLOT(pdev->devfn), - (unsigned int)PCI_FUNC(pdev->devfn), - ret); + ret = pci_enable_device(dev); + + if (ret != 0) { + mods_error_printk("failed to enable device %04x:%x:%02x.%x\n", + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + return 0; } - return ret; + + dpriv = kzalloc(sizeof(*dpriv), GFP_KERNEL | __GFP_NORETRY); + if (unlikely(!dpriv)) + return 0; + dpriv->client_id = client->client_id; + dpriv->dev = dev; + dpriv->next = client->enabled_devices; + client->enabled_devices = dpriv; + pci_set_drvdata(dev, dpriv); + + return dpriv; } -void mods_disable_device(struct pci_dev *pdev) +void mods_disable_device(struct pci_dev *dev) { - struct en_dev_entry *dpriv = pci_get_drvdata(pdev); + struct en_dev_entry *dpriv = pci_get_drvdata(dev); + + BUG_ON(!mutex_is_locked(&mp.mtx)); if (dpriv) - pci_set_drvdata(pdev, NULL); + pci_set_drvdata(dev, NULL); - pci_disable_device(pdev); + pci_disable_device(dev); } #endif @@ -126,14 +120,6 @@ static unsigned int get_cur_time(void) return jiffies_to_usecs(jiffies); } -static int id_is_valid(unsigned char channel) -{ - if (channel <= 0 || channel > MODS_CHANNEL_MAX) - return -EINVAL; - - return OK; -} - static inline int mods_check_interrupt(struct dev_irq_map *t) { int ii = 0; @@ -193,6 +179,7 @@ static void mods_disable_interrupts(struct dev_irq_map *t) } } +#ifdef CONFIG_PCI static const char *mods_irq_type_name(u8 irq_type) { switch (irq_type) { @@ -208,18 +195,23 @@ static const char *mods_irq_type_name(u8 irq_type) return "unknown"; } } +#endif -static void rec_irq_done(struct nv_device *dev, - unsigned char channel, - struct dev_irq_map *t, +static void wake_up_client(struct dev_irq_map *t) +{ + struct mods_client *client = &mp.clients[t->client_id - 1]; + + if (client) + wake_up_interruptible(&client->interrupt_event); +} + +static int rec_irq_done(struct dev_irq_map *t, unsigned int irq_time) { struct irq_q_info *q; - struct mods_priv *pmp = dev->isr_pri; - struct mods_file_private_data *private_data = dev->pri[channel - 1]; /* Get interrupt queue */ - q = &pmp->rec_info[channel - 1]; + q = &mp.clients[t->client_id - 1].irq_queue; /* Don't do anything if the IRQ has already been recorded */ if (q->head != q->tail) { @@ -231,7 +223,7 @@ static void rec_irq_done(struct nv_device *dev, if ((pd->irq == t->apic_irq) && (!t->dev || (pd->dev == t->dev))) - return; + return false; } } @@ -239,7 +231,7 @@ static void rec_irq_done(struct nv_device *dev, /* This is deadly! */ if (q->tail - q->head == MODS_MAX_IRQS) { mods_error_printk("IRQ queue is full\n"); - return; + return false; } /* Record the device which generated the IRQ in the queue */ @@ -267,12 +259,7 @@ static void rec_irq_done(struct nv_device *dev, t->apic_irq, irq_time); - /* Wake MODS to handle the interrupt */ - if (private_data) { - spin_unlock(&pmp->lock); - wake_up_interruptible(&private_data->interrupt_event); - spin_lock(&pmp->lock); - } + return true; } /* mods_irq_handle - interrupt function */ @@ -282,75 +269,72 @@ static irqreturn_t mods_irq_handle(int irq, void *data #endif ) { - struct nv_device *dev = (struct nv_device *)data; - struct mods_priv *pmp = dev->isr_pri; - struct dev_irq_map *t = NULL; - unsigned char channel_idx; - unsigned long flags = 0; - int found = 0; - unsigned int irq_time = get_cur_time(); + struct dev_irq_map *t = (struct dev_irq_map *)data; + int serviced = false; - spin_lock_irqsave(&pmp->lock, flags); + if (unlikely(!t)) + mods_error_printk("received irq %d, but no context for it\n", + irq); + else if (unlikely(t->apic_irq != irq)) + mods_error_printk("received irq %d which doesn't match registered irq %d\n", + irq, t->apic_irq); + else { + unsigned long flags = 0; + int recorded = false; + unsigned int irq_time = get_cur_time(); + struct mods_client *client = &mp.clients[t->client_id - 1]; - for (channel_idx = 0; channel_idx < MODS_CHANNEL_MAX; channel_idx++) { - if (!(pmp->channel_flags & (1 << channel_idx))) - continue; + spin_lock_irqsave(&client->irq_lock, flags); - list_for_each_entry(t, &pmp->irq_head[channel_idx], list) { - if ((t->apic_irq == irq) && mods_check_interrupt(t)) { - /* Disable interrupts on this device to avoid - * interrupt storm - */ - mods_disable_interrupts(t); + /* Check if the interrupt is still pending (shared INTA) */ + if (mods_check_interrupt(t)) { - /* Record IRQ for MODS and wake MODS up */ - rec_irq_done(dev, channel_idx+1, t, irq_time); - found |= 1; + /* Disable interrupts on this device to avoid interrupt + * storm + */ + mods_disable_interrupts(t); - /* MSI, MSI-X and CPU interrupts are not shared, - * so stop looking - */ - if (t->type != MODS_IRQ_TYPE_INT) { - channel_idx = MODS_CHANNEL_MAX; - break; - } - } + /* Record IRQ for MODS and wake MODS up */ + recorded = rec_irq_done(t, irq_time); + + serviced = true; } + + spin_unlock_irqrestore(&client->irq_lock, flags); + + if (recorded) + wake_up_client(t); } - spin_unlock_irqrestore(&pmp->lock, flags); - return IRQ_RETVAL(found); + return IRQ_RETVAL(serviced); } -static int mods_lookup_irq(unsigned char channel, struct pci_dev *pdev, - unsigned int irq) +static int mods_lookup_cpu_irq(u8 client_id, unsigned int irq) { - unsigned char channel_idx; - struct mods_priv *pmp = get_all_data(); + u8 client_idx; int ret = IRQ_NOT_FOUND; LOG_ENT(); - for (channel_idx = 0; channel_idx < MODS_CHANNEL_MAX; channel_idx++) { + for (client_idx = 0; client_idx < MODS_MAX_CLIENTS; client_idx++) { struct dev_irq_map *t = NULL; struct dev_irq_map *next = NULL; list_for_each_entry_safe(t, next, - &pmp->irq_head[channel_idx], + &mp.clients[client_idx].irq_list, list) { - if ((t->apic_irq == irq) && - (!pdev || (t->dev == pdev))) { - if (channel == 0) { + if (t->apic_irq == irq) { + if (client_id == 0) { ret = IRQ_FOUND; } else { - ret = (channel == channel_idx + 1) + ret = (client_id == client_idx + 1) ? IRQ_FOUND : IRQ_NOT_FOUND; } /* Break out of the outer loop */ - channel_idx = MODS_CHANNEL_MAX; + client_idx = MODS_MAX_CLIENTS; break; } } @@ -361,18 +345,17 @@ static int mods_lookup_irq(unsigned char channel, struct pci_dev *pdev, } #ifdef CONFIG_PCI -static int is_nvidia_device(struct pci_dev *pdev) +static int is_nvidia_gpu(struct pci_dev *dev) { - unsigned short class_code, vendor_id, device_id; + unsigned short class_code, vendor_id; - pci_read_config_word(pdev, PCI_CLASS_DEVICE, &class_code); - pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor_id); - pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); + pci_read_config_word(dev, PCI_CLASS_DEVICE, &class_code); + pci_read_config_word(dev, PCI_VENDOR_ID, &vendor_id); if (((class_code == PCI_CLASS_DISPLAY_VGA) || (class_code == PCI_CLASS_DISPLAY_3D)) && (vendor_id == 0x10DE)) { - return 1; + return true; } - return 0; + return false; } #endif @@ -385,7 +368,7 @@ static void setup_mask_info(struct dev_irq_map *newmap, char *bar = newmap->dev_irq_aperture; u32 ii = 0; - if ((p->mask_info_cnt == 0) && is_nvidia_device(pdev)) { + if ((p->mask_info_cnt == 0) && is_nvidia_gpu(pdev)) { newmap->mask_info_cnt = 1; newmap->mask_info[0].dev_irq_mask_reg = (u32 *)(bar+0x140); newmap->mask_info[0].dev_irq_disable_reg = (u32 *)(bar+0x140); @@ -410,14 +393,12 @@ static void setup_mask_info(struct dev_irq_map *newmap, } #endif -static int add_irq_map(unsigned char channel, - struct pci_dev *pdev, - struct MODS_REGISTER_IRQ_4 *p, u32 irq, u32 entry) +static int add_irq_map(u8 client_id, + struct pci_dev *pdev, + struct MODS_REGISTER_IRQ_4 *p, u32 irq, u32 entry) { u32 irq_type = MODS_IRQ_TYPE_FROM_FLAGS(p->irq_flags); struct dev_irq_map *newmap = NULL; - struct mods_priv *pmp = get_all_data(); - struct nv_device *nvdev = get_dev(); LOG_ENT(); @@ -431,7 +412,7 @@ static int add_irq_map(unsigned char channel, /* Fill out the new entry */ newmap->apic_irq = irq; newmap->dev = pdev; - newmap->channel = channel; + newmap->client_id = client_id; newmap->dev_irq_aperture = 0; newmap->mask_info_cnt = 0; newmap->type = irq_type; @@ -442,8 +423,8 @@ static int add_irq_map(unsigned char channel, irq, &mods_irq_handle, (irq_type == MODS_IRQ_TYPE_INT) ? IRQF_SHARED : 0, - nvdev->name, - nvdev)) { + "nvidia mods", + newmap)) { mods_error_printk("unable to enable IRQ 0x%x\n", irq); kfree(newmap); LOG_EXT(); @@ -451,7 +432,7 @@ static int add_irq_map(unsigned char channel, } /* Add the new entry to the list of all registered interrupts */ - list_add(&newmap->list, &pmp->irq_head[channel - 1]); + list_add(&newmap->list, &mp.clients[client_id - 1].irq_list); #ifdef CONFIG_PCI /* Map BAR0 to be able to disable interrupts */ @@ -530,18 +511,23 @@ static int add_irq_map(unsigned char channel, static void mods_free_map(struct dev_irq_map *del) { + unsigned long flags = 0; + struct mods_client *client = &mp.clients[del->client_id - 1]; + LOG_ENT(); /* Disable interrupts on the device */ + spin_lock_irqsave(&client->irq_lock, flags); mods_disable_interrupts(del); + spin_unlock_irqrestore(&client->irq_lock, flags); + + /* Unhook interrupts in the kernel */ + free_irq(del->apic_irq, del); /* Unmap aperture used for masking irqs */ if (del->dev_irq_aperture) iounmap(del->dev_irq_aperture); - /* Unhook interrupts in the kernel */ - free_irq(del->apic_irq, get_dev()); - /* Free memory */ kfree(del); @@ -550,53 +536,33 @@ static void mods_free_map(struct dev_irq_map *del) void mods_init_irq(void) { - int i; - struct mods_priv *pmp = get_all_data(); - LOG_ENT(); - memset(pmp, 0, sizeof(struct mods_priv)); - for (i = 0; i < MODS_CHANNEL_MAX; i++) - INIT_LIST_HEAD(&pmp->irq_head[i]); + memset(&mp, 0, sizeof(mp)); + + mutex_init(&mp.mtx); - spin_lock_init(&pmp->lock); LOG_EXT(); } void mods_cleanup_irq(void) { int i; - struct mods_priv *pmp = get_all_data(); LOG_ENT(); - for (i = 0; i < MODS_CHANNEL_MAX; i++) { - if (pmp->channel_flags && (1 << i)) - mods_free_channel(i + 1); + for (i = 0; i < MODS_MAX_CLIENTS; i++) { + if (mp.client_flags && (1 << i)) + mods_free_client(i + 1); } LOG_EXT(); } -void mods_irq_dev_set_pri(unsigned char id, void *pri) +int mods_irq_event_check(u8 client_id) { - struct nv_device *dev = get_dev(); + struct irq_q_info *q = &mp.clients[client_id - 1].irq_queue; + unsigned int pos = (1 << (client_id - 1)); - dev->pri[id - 1] = pri; -} - -void mods_irq_dev_clr_pri(unsigned char id) -{ - struct nv_device *dev = get_dev(); - - dev->pri[id - 1] = 0; -} - -int mods_irq_event_check(unsigned char channel) -{ - struct mods_priv *pmp = get_all_data(); - struct irq_q_info *q = &pmp->rec_info[channel - 1]; - unsigned int pos = (1 << (channel - 1)); - - if (!(pmp->channel_flags & pos)) + if (!(mp.client_flags & pos)) return POLLERR; /* irq has quit */ if (q->head != q->tail) @@ -605,65 +571,100 @@ int mods_irq_event_check(unsigned char channel) return 0; } -unsigned char mods_alloc_channel(void) +struct mods_client *mods_alloc_client(void) { - struct mods_priv *pmp = get_all_data(); - int i = 0; - unsigned char channel = MODS_CHANNEL_MAX + 1; - unsigned char max_channels = 1; + u8 idx = 0; + u8 max_clients = 1; LOG_ENT(); if (mods_get_multi_instance() || (mods_get_access_token() != MODS_ACCESS_TOKEN_NONE)) - max_channels = MODS_CHANNEL_MAX; + max_clients = MODS_MAX_CLIENTS; + + for (idx = 0; idx < max_clients; idx++) { + if (!test_and_set_bit(idx, &mp.client_flags)) { + struct mods_client *client = &mp.clients[idx]; - for (i = 0; i < max_channels; i++) { - if (!test_and_set_bit(i, &pmp->channel_flags)) { - channel = i + 1; mods_debug_printk(DEBUG_IOCTL, - "open channel %u (bit mask 0x%lx)\n", - (unsigned int)(i + 1), - pmp->channel_flags); - break; - } + "open client %u (bit mask 0x%lx)\n", + (unsigned int)(idx + 1), + mp.client_flags); + memset(client, 0, sizeof(*client)); + client->client_id = idx + 1; + client->access_token = MODS_ACCESS_TOKEN_NONE; + mutex_init(&client->mtx); + spin_lock_init(&client->irq_lock); + init_waitqueue_head(&client->interrupt_event); + INIT_LIST_HEAD(&client->irq_list); + INIT_LIST_HEAD(&client->mem_alloc_list); + INIT_LIST_HEAD(&client->mem_map_list); +#if defined(CONFIG_PPC64) + INIT_LIST_HEAD(&client->ppc_tce_bypass_list); + INIT_LIST_HEAD(&client->nvlink_sysmem_trained_list); +#endif + + LOG_EXT(); + return client; + } } LOG_EXT(); - return channel; + return NULL; } -static int mods_free_irqs(unsigned char channel, struct pci_dev *dev) +static int mods_free_irqs(u8 client_id, struct pci_dev *dev) { #ifdef CONFIG_PCI - struct mods_priv *pmp = get_all_data(); - struct dev_irq_map *del = NULL; - struct dev_irq_map *next; - struct en_dev_entry *dpriv = pci_get_drvdata(dev); - unsigned int irq_type; + struct mods_client *client = &mp.clients[client_id - 1]; + struct dev_irq_map *del = NULL; + struct dev_irq_map *next; + struct en_dev_entry *dpriv; + unsigned int irq_type; LOG_ENT(); - if (!dpriv || !dpriv->irqs_allocated) { + if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + LOG_EXT(); + return -EINTR; + } + + dpriv = pci_get_drvdata(dev); + + if (!dpriv) { + mutex_unlock(&mp.mtx); LOG_EXT(); return OK; } + if (dpriv->client_id != client_id) { + mods_error_printk("invalid client %u for device %04x:%x:%02x.%x\n", + (unsigned int)client_id, + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + mutex_unlock(&mp.mtx); + LOG_EXT(); + return -EINVAL; + } + mods_debug_printk(DEBUG_ISR_DETAILED, - "(dev=%04x:%x:%02x.%x) irqs_allocated=%d irq_flags=0x%x nvecs=%d\n", + "(dev=%04x:%x:%02x.%x) irq_flags=0x%x nvecs=%d\n", pci_domain_nr(dev->bus), dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), - dpriv->irqs_allocated, dpriv->irq_flags, dpriv->nvecs ); /* Delete device interrupts from the list */ - list_for_each_entry_safe(del, next, &pmp->irq_head[channel - 1], list) { + list_for_each_entry_safe(del, next, &client->irq_list, list) { if (dev == del->dev) { + u8 type = del->type; + list_del(&del->list); mods_debug_printk(DEBUG_ISR, "%04x:%x:%02x.%x unregistered %s IRQ 0x%x\n", @@ -671,11 +672,13 @@ static int mods_free_irqs(unsigned char channel, struct pci_dev *dev) dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn), - mods_irq_type_name(del->type), + mods_irq_type_name(type), del->apic_irq); mods_free_map(del); - if (del->type != MODS_IRQ_TYPE_MSIX) + BUG_ON(type != + MODS_IRQ_TYPE_FROM_FLAGS(dpriv->irq_flags)); + if (type != MODS_IRQ_TYPE_MSIX) break; } } @@ -687,99 +690,81 @@ static int mods_free_irqs(unsigned char channel, struct pci_dev *dev) if (irq_type == MODS_IRQ_TYPE_MSIX) { pci_disable_msix(dev); kfree(dpriv->msix_entries); + dpriv->msix_entries = 0; } else if (irq_type == MODS_IRQ_TYPE_MSI) { pci_disable_msi(dev); } #endif dpriv->nvecs = 0; - dpriv->msix_entries = NULL; - dpriv->irqs_allocated = false; mods_debug_printk(DEBUG_ISR_DETAILED, "irqs freed\n"); #endif + mutex_unlock(&mp.mtx); LOG_EXT(); return 0; } -void mods_free_channel(unsigned char channel) +void mods_free_client_interrupts(struct mods_client *client) { - struct nv_device *nvdev = get_dev(); - MODS_PRIV priv = nvdev->pri[channel - 1]; - struct en_dev_entry *dent = priv->enabled_devices; - struct mods_priv *pmp = get_all_data(); - struct irq_q_info *q = &pmp->rec_info[channel - 1]; + struct en_dev_entry *dpriv = client->enabled_devices; LOG_ENT(); /* Release all interrupts */ - while (dent) { - mods_free_irqs(priv->mods_id, dent->dev); - dent = dent->next; + while (dpriv) { + mods_free_irqs(client->client_id, dpriv->dev); + dpriv = dpriv->next; } - /* Clear queue */ - memset(q, 0, sizeof(*q)); + LOG_EXT(); +} - /* Indicate the channel is free */ - clear_bit(channel - 1, &pmp->channel_flags); +void mods_free_client(u8 client_id) +{ + struct mods_client *client = &mp.clients[client_id - 1]; - mods_debug_printk(DEBUG_IOCTL, "closed channel %u\n", - (unsigned int)channel); + LOG_ENT(); + + memset(client, 0, sizeof(*client)); + + /* Indicate the client_id is free */ + clear_bit(client_id - 1, &mp.client_flags); + + mods_debug_printk(DEBUG_IOCTL, "closed client %u\n", + (unsigned int)client_id); LOG_EXT(); } #ifdef CONFIG_PCI -static int mods_allocate_irqs(struct file *pfile, struct mods_pci_dev_2 *bdf, - __u32 nvecs, __u32 flags) +static int mods_allocate_irqs(u8 client_id, struct file *pfile, + struct pci_dev *dev, u32 nvecs, u32 flags) { - MODS_PRIV private_data = pfile->private_data; - struct pci_dev *dev; + struct mods_client *client = pfile->private_data; struct en_dev_entry *dpriv; - unsigned int devfn; - unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(flags); + unsigned int irq_type = MODS_IRQ_TYPE_FROM_FLAGS(flags); LOG_ENT(); mods_debug_printk(DEBUG_ISR_DETAILED, "(dev=%04x:%x:%02x.%x, flags=0x%x, nvecs=%d)\n", - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function, + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn), flags, nvecs); - if (!nvecs) { - mods_error_printk("no irq's requested!\n"); - LOG_EXT(); - return -EINVAL; - } - - /* Get the PCI device structure for the specified device from kernel */ - devfn = PCI_DEVFN(bdf->device, bdf->function); - dev = MODS_PCI_GET_SLOT(bdf->domain, bdf->bus, devfn); - if (!dev) { - mods_error_printk( - "unknown dev %04x:%x:%02x.%x\n", - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function); - LOG_EXT(); - return -EINVAL; - } - /* Determine if the device supports requested interrupt type */ if (irq_type == MODS_IRQ_TYPE_MSI) { #ifdef CONFIG_PCI_MSI if (pci_find_capability(dev, PCI_CAP_ID_MSI) == 0) { mods_error_printk( "dev %04x:%x:%02x.%x does not support MSI\n", - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function); + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); LOG_EXT(); return -EINVAL; } @@ -789,41 +774,25 @@ static int mods_allocate_irqs(struct file *pfile, struct mods_pci_dev_2 *bdf, #endif } else if (irq_type == MODS_IRQ_TYPE_MSIX) { #ifdef CONFIG_PCI_MSI - int msix_cap; - - msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); - - if (!msix_cap) { + if (pci_find_capability(dev, PCI_CAP_ID_MSIX) == 0) { mods_error_printk( "dev %04x:%x:%02x.%x does not support MSI-X\n", - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function); + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); LOG_EXT(); return -EINVAL; } #else - mods_error_printk("the kernel does not support MSIX!\n"); + mods_error_printk("the kernel does not support MSI-X!\n"); return -EINVAL; #endif } /* Enable device on the PCI bus */ - if (mods_enable_device(private_data, dev)) { - mods_error_printk("unable to enable dev %04x:%x:%02x.%x\n", - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function); - LOG_EXT(); - return -EINVAL; - } - - dpriv = pci_get_drvdata(dev); - + dpriv = mods_enable_device(client, dev); if (!dpriv) { - mods_error_printk("could not get mods dev private!\n"); LOG_EXT(); return -EINVAL; } @@ -850,10 +819,11 @@ static int mods_allocate_irqs(struct file *pfile, struct mods_pci_dev_2 *bdf, if (pci_enable_msi(dev) != 0) { mods_error_printk( "unable to enable MSI on dev %04x:%x:%02x.%x\n", - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function); + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + LOG_EXT(); return -EINVAL; } dpriv->nvecs = 1; @@ -921,26 +891,22 @@ static int mods_allocate_irqs(struct file *pfile, struct mods_pci_dev_2 *bdf, } #endif else { - mods_error_printk( - "unsupported irq_type %d dev: %04x:%x:%02x.%x\n", - irq_type, - (unsigned int)bdf->domain, - (unsigned int)bdf->bus, - (unsigned int)bdf->device, - (unsigned int)bdf->function); - LOG_EXT(); - return -EINVAL; + mods_error_printk("unsupported irq_type %d dev: %04x:%x:%02x.%x\n", + irq_type, + pci_domain_nr(dev->bus), + dev->bus->number, + PCI_SLOT(dev->devfn), + PCI_FUNC(dev->devfn)); + LOG_EXT(); + return -EINVAL; } - dpriv->irqs_allocated = true; + dpriv->client_id = client_id; dpriv->irq_flags = flags; LOG_EXT(); return OK; } -static int mods_unregister_pci_irq(struct file *pfile, - struct MODS_REGISTER_IRQ_2 *p); - static int mods_register_pci_irq(struct file *pfile, struct MODS_REGISTER_IRQ_4 *p) { @@ -949,15 +915,15 @@ static int mods_register_pci_irq(struct file *pfile, struct pci_dev *dev; struct en_dev_entry *dpriv; unsigned int devfn; - unsigned char channel; + u8 client_id; int i; LOG_ENT(); /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { + client_id = get_client_id(pfile); + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } @@ -976,46 +942,72 @@ static int mods_register_pci_irq(struct file *pfile, return -EINVAL; } - dpriv = pci_get_drvdata(dev); - - if (!dpriv || !dpriv->irqs_allocated) { - if (mods_allocate_irqs(pfile, &p->dev, p->irq_count, - p->irq_flags)) { - mods_error_printk( - "could not allocate irqs for irq_type %d\n", - irq_type); - LOG_EXT(); - return -EINVAL; - } - dpriv = pci_get_drvdata(dev); - } else { - mods_error_printk( - "dev %04x:%x:%02x.%x has already been registered\n", - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function); + if (!p->irq_count) { + mods_error_printk("no irq's requested!\n"); LOG_EXT(); return -EINVAL; } + if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + LOG_EXT(); + return -EINTR; + } + + dpriv = pci_get_drvdata(dev); + if (dpriv) { + if (dpriv->client_id != client_id) { + mods_error_printk("dev %04x:%x:%02x.%x already owned by client %u\n", + (unsigned int)p->dev.domain, + (unsigned int)p->dev.bus, + (unsigned int)p->dev.device, + (unsigned int)p->dev.function, + (unsigned int)dpriv->client_id); + mutex_unlock(&mp.mtx); + LOG_EXT(); + return -EINVAL; + } + if (dpriv->nvecs) { + mods_error_printk("interrupt for dev %04x:%x:%02x.%x already registered\n", + (unsigned int)p->dev.domain, + (unsigned int)p->dev.bus, + (unsigned int)p->dev.device, + (unsigned int)p->dev.function); + mutex_unlock(&mp.mtx); + LOG_EXT(); + return -EINVAL; + } + } + + if (mods_allocate_irqs(client_id, pfile, dev, p->irq_count, + p->irq_flags)) { + mods_error_printk("could not allocate irqs for irq_type %d\n", + irq_type); + mutex_unlock(&mp.mtx); + LOG_EXT(); + return -EINVAL; + } + + dpriv = pci_get_drvdata(dev); + for (i = 0; i < p->irq_count; i++) { u32 irq = ((irq_type == MODS_IRQ_TYPE_INT) || (irq_type == MODS_IRQ_TYPE_MSI)) ? dev->irq : dpriv->msix_entries[i].vector; - if (add_irq_map(channel, dev, p, irq, i) != OK) { + if (add_irq_map(client_id, dev, p, irq, i) != OK) { #ifdef CONFIG_PCI_MSI if (irq_type == MODS_IRQ_TYPE_MSI) pci_disable_msi(dev); else if (irq_type == MODS_IRQ_TYPE_MSIX) pci_disable_msix(dev); #endif + mutex_unlock(&mp.mtx); LOG_EXT(); return -EINVAL; } } + mutex_unlock(&mp.mtx); LOG_EXT(); return rc; } @@ -1024,33 +1016,41 @@ static int mods_register_pci_irq(struct file *pfile, static int mods_register_cpu_irq(struct file *pfile, struct MODS_REGISTER_IRQ_4 *p) { - unsigned char channel; + u8 client_id; u32 irq = p->dev.bus; LOG_ENT(); /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { + client_id = get_client_id(pfile); + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } + if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + LOG_EXT(); + return -EINTR; + } + /* Determine if the interrupt is already hooked */ - if (mods_lookup_irq(0, 0, irq) == IRQ_FOUND) { + if (mods_lookup_cpu_irq(0, irq) == IRQ_FOUND) { mods_error_printk("CPU IRQ 0x%x has already been registered\n", irq); + mutex_unlock(&mp.mtx); LOG_EXT(); return -EINVAL; } /* Register interrupt */ - if (add_irq_map(channel, 0, p, irq, 0) != OK) { + if (add_irq_map(client_id, 0, p, irq, 0) != OK) { + mutex_unlock(&mp.mtx); LOG_EXT(); return -EINVAL; } + mutex_unlock(&mp.mtx); return OK; } @@ -1060,15 +1060,15 @@ static int mods_unregister_pci_irq(struct file *pfile, { struct pci_dev *dev; unsigned int devfn; - unsigned char channel; + u8 client_id; int rv = OK; LOG_ENT(); /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { + client_id = get_client_id(pfile); + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } @@ -1081,7 +1081,7 @@ static int mods_unregister_pci_irq(struct file *pfile, return -EINVAL; } - rv = mods_free_irqs(channel, dev); + rv = mods_free_irqs(client_id, dev); LOG_EXT(); return rv; @@ -1091,38 +1091,46 @@ static int mods_unregister_pci_irq(struct file *pfile, static int mods_unregister_cpu_irq(struct file *pfile, struct MODS_REGISTER_IRQ_2 *p) { - struct mods_priv *pmp = get_all_data(); struct dev_irq_map *del = NULL; struct dev_irq_map *next; - unsigned int irq; - unsigned char channel; + unsigned int irq; + u8 client_id; + struct mods_client *client; LOG_ENT(); irq = p->dev.bus; /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { + client_id = get_client_id(pfile); + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } + client = &mp.clients[client_id = 1]; + + if (unlikely(mutex_lock_interruptible(&mp.mtx))) { + LOG_EXT(); + return -EINTR; + } /* Determine if the interrupt is already hooked by this client */ - if (mods_lookup_irq(channel, 0, irq) == IRQ_NOT_FOUND) { + if (mods_lookup_cpu_irq(client_id, irq) == IRQ_NOT_FOUND) { mods_error_printk( "IRQ 0x%x not hooked, can't unhook\n", irq); + mutex_unlock(&mp.mtx); LOG_EXT(); return -EINVAL; } /* Delete device interrupt from the list */ - list_for_each_entry_safe(del, next, &pmp->irq_head[channel - 1], list) { + list_for_each_entry_safe(del, next, &client->irq_list, list) { if ((irq == del->apic_irq) && (del->dev == 0)) { if (del->type != p->type) { mods_error_printk("wrong IRQ type passed\n"); + mutex_unlock(&mp.mtx); LOG_EXT(); return -EINVAL; } @@ -1135,6 +1143,7 @@ static int mods_unregister_cpu_irq(struct file *pfile, } } + mutex_unlock(&mp.mtx); LOG_EXT(); return OK; } @@ -1197,34 +1206,25 @@ int esc_mods_register_irq_2(struct file *pfile, irq_data.dev = p->dev; irq_data.irq_count = 1; irq_data.irq_flags = p->type; - if (p->type == MODS_IRQ_TYPE_CPU) - return mods_register_cpu_irq(pfile, &irq_data); + #ifdef CONFIG_PCI { + /* Get the PCI device structure */ + unsigned int devfn; + struct pci_dev *dev; - /* Get the PCI device structure for the specified device from kernel */ - unsigned int devfn; - struct pci_dev *dev; - - devfn = PCI_DEVFN(p->dev.device, p->dev.function); - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { - LOG_EXT(); - return -EINVAL; - } - /* initialize the new entry to behave like old implementation */ - irq_data.mask_info_cnt = 0; - if ((p->type == MODS_IRQ_TYPE_INT) && is_nvidia_device(dev)) { + devfn = PCI_DEVFN(p->dev.device, p->dev.function); + dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); + if (!dev) { + LOG_EXT(); + return -EINVAL; + } irq_data.aperture_addr = pci_resource_start(dev, 0); - irq_data.aperture_size = 0x200; + irq_data.aperture_size = pci_resource_len(dev, 0); } - - return mods_register_pci_irq(pfile, &irq_data); - } -#else - mods_error_printk("PCI not available\n"); - return -EINVAL; #endif + + return esc_mods_register_irq_4(pfile, &irq_data); } int esc_mods_register_irq(struct file *pfile, @@ -1269,30 +1269,32 @@ int esc_mods_unregister_irq(struct file *pfile, int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) { - unsigned char channel; - struct irq_q_info *q = NULL; - struct mods_priv *pmp = get_all_data(); - unsigned int i = 0; - unsigned long flags = 0; - unsigned int cur_time = get_cur_time(); + u8 client_id; + struct mods_client *client; + struct irq_q_info *q = NULL; + unsigned int i = 0; + unsigned long flags = 0; + unsigned int cur_time = get_cur_time(); - /* Lock IRQ queue */ LOG_ENT(); - spin_lock_irqsave(&pmp->lock, flags); /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { + client_id = get_client_id(pfile); + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } + client = &mp.clients[client_id - 1]; /* Clear return array */ memset(p->irq_list, 0xFF, sizeof(p->irq_list)); + /* Lock IRQ queue */ + spin_lock_irqsave(&client->irq_lock, flags); + /* Fill in return array with IRQ information */ - q = &pmp->rec_info[channel - 1]; + q = &client->irq_queue; for (i = 0; (q->head != q->tail) && (i < MODS_MAX_IRQS); q->head++, i++) { @@ -1339,9 +1341,9 @@ int esc_mods_query_irq_3(struct file *pfile, struct MODS_QUERY_IRQ_3 *p) p->more = 1; /* Unlock IRQ queue */ - spin_unlock_irqrestore(&pmp->lock, flags); - LOG_EXT(); + spin_unlock_irqrestore(&client->irq_lock, flags); + LOG_EXT(); return OK; } @@ -1383,233 +1385,39 @@ int esc_mods_query_irq(struct file *pfile, return OK; } -int esc_mods_set_irq_multimask(struct file *pfile, - struct MODS_SET_IRQ_MULTIMASK *p) -{ - struct mods_priv *pmp = get_all_data(); - struct pci_dev *dev = 0; - unsigned long flags = 0; - unsigned char channel; - u32 irq = ~0U; - int ret = -EINVAL; - int update_state_reg = 0; - - LOG_ENT(); - - /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { - LOG_EXT(); - return -EINVAL; - } - - /* Print info */ - if (p->irq_type == MODS_IRQ_TYPE_CPU) { - mods_debug_printk( - DEBUG_ISR, - "set CPU IRQ 0x%x mask &0x%llx |0x%llx addr=0x%llx\n", - (unsigned int)p->dev.bus, - p->mask_info[0].and_mask, p->mask_info[0].or_mask, - p->aperture_addr + p->mask_info[0].reg_offset); - } else { - mods_debug_printk( - DEBUG_ISR, -"set %s IRQ mask dev %04x:%x:%02x.%x mask t%d &0x%llx |0x%llx addr=0x%llx\n", - mods_irq_type_name(p->irq_type), - (unsigned int)p->dev.domain, - (unsigned int)p->dev.bus, - (unsigned int)p->dev.device, - (unsigned int)p->dev.function, - p->mask_info[0].mask_type, - p->mask_info[0].and_mask, - p->mask_info[0].or_mask, - p->aperture_addr + p->mask_info[0].reg_offset); - } - - /* Verify mask type */ - if (p->mask_info[0].mask_type != MODS_MASK_TYPE_IRQ_DISABLE && - p->mask_info[0].mask_type != MODS_MASK_TYPE_IRQ_DISABLE64) { - mods_error_printk("invalid mask type\n"); - LOG_EXT(); - return -EINVAL; - } - - /* Determine which interrupt is referenced */ - if (p->irq_type == MODS_IRQ_TYPE_CPU) - irq = p->dev.bus; - else if (p->irq_type == MODS_IRQ_TYPE_INT) { -#ifdef CONFIG_PCI - /* Get the PCI dev struct for the specified device from kernel*/ - unsigned int devfn = PCI_DEVFN(p->dev.device, p->dev.function); - - dev = MODS_PCI_GET_SLOT(p->dev.domain, p->dev.bus, devfn); - if (!dev) { - LOG_EXT(); - return -EINVAL; - } - update_state_reg = is_nvidia_device(dev); - if (!update_state_reg) { - mods_error_printk( - "set_irq_multimask is only supported GPUs!\n"); - LOG_EXT(); - return -EINVAL; - } -#else - mods_error_printk("PCI not available\n"); - LOG_EXT(); - return -EINVAL; -#endif - } else { - mods_error_printk("set_irq_multimask not supported for %s!\n", - mods_irq_type_name(p->irq_type)); - LOG_EXT(); - return -EINVAL; - } - - /* Lock IRQ queue */ - spin_lock_irqsave(&pmp->lock, flags); - { - struct dev_irq_map *t = NULL; - struct dev_irq_map *next = NULL; - struct irq_mask_info *im = NULL; - struct mods_mask_info *mm = NULL; - - list_for_each_entry_safe(t, next, &pmp->irq_head[channel-1], list) { - if ((dev && (t->dev == dev)) || - (!dev && (t->apic_irq == irq))) { - u32 i = 0; - char *bar = 0; - - if (t->type != p->irq_type) { - mods_error_printk( - "IRQ type does not match registered IRQ\n"); - break; - } - if (t->dev_irq_aperture) { - iounmap(t->dev_irq_aperture); - t->dev_irq_aperture = 0; - t->mask_info_cnt = 0; - t->mask_info[0].dev_irq_mask_reg = 0; - t->mask_info[0].dev_irq_disable_reg = 0; - t->mask_info[0].dev_irq_state = 0; - mods_warning_printk("resetting IRQ mask\n"); - } - bar = ioremap_nocache(p->aperture_addr, - p->aperture_size); - if (!bar) { - mods_error_printk( - "unable to remap specified aperture\n"); - break; - } - - t->dev_irq_aperture = bar; - t->mask_info_cnt = p->mask_info_cnt; - for (i = 0; i < p->mask_info_cnt; i++) { - im = &t->mask_info[i]; - mm = &p->mask_info[i]; - im->dev_irq_disable_reg = - (u32 *)(bar + mm->reg_offset); - im->dev_irq_mask_reg = - (u32 *)(bar + mm->reg_offset); - im->irq_and_mask = mm->and_mask; - im->irq_or_mask = mm->or_mask; - im->mask_type = mm->mask_type; - if (update_state_reg) { - im->dev_irq_state = - (u32 *)(bar+0x100); - im->dev_irq_mask_reg = - (u32 *)(bar+0x140); - } else { - im->dev_irq_state = 0; - im->dev_irq_mask_reg = 0; - } - } - ret = OK; - break; - } - } - } - /* Unlock IRQ queue */ - spin_unlock_irqrestore(&pmp->lock, flags); - LOG_EXT(); - - return ret; -} - -int esc_mods_set_irq_mask_2(struct file *pfile, - struct MODS_SET_IRQ_MASK_2 *p) -{ - struct MODS_SET_IRQ_MULTIMASK set_irq_multimask = {0}; - - set_irq_multimask.aperture_addr = p->aperture_addr; - set_irq_multimask.aperture_size = p->aperture_size; - set_irq_multimask.mask_info_cnt = 1; - set_irq_multimask.mask_info[0].reg_offset = p->reg_offset; - set_irq_multimask.mask_info[0].and_mask = p->and_mask; - set_irq_multimask.mask_info[0].or_mask = p->or_mask; - set_irq_multimask.mask_info[0].mask_type = p->mask_type; - set_irq_multimask.dev.domain = p->dev.domain; - set_irq_multimask.dev.bus = p->dev.bus; - set_irq_multimask.dev.device = p->dev.device; - set_irq_multimask.dev.function = p->dev.function; - set_irq_multimask.irq_type = p->irq_type; - - return esc_mods_set_irq_multimask(pfile, &set_irq_multimask); -} - -int esc_mods_set_irq_mask(struct file *pfile, - struct MODS_SET_IRQ_MASK *p) -{ - struct MODS_SET_IRQ_MULTIMASK set_irq_multimask = {0}; - - set_irq_multimask.aperture_addr = p->aperture_addr; - set_irq_multimask.aperture_size = p->aperture_size; - set_irq_multimask.mask_info_cnt = 1; - set_irq_multimask.mask_info[0].reg_offset = p->reg_offset; - set_irq_multimask.mask_info[0].and_mask = p->and_mask; - set_irq_multimask.mask_info[0].or_mask = p->or_mask; - set_irq_multimask.mask_info[0].mask_type = p->mask_type; - set_irq_multimask.dev.domain = 0; - set_irq_multimask.dev.bus = p->dev.bus; - set_irq_multimask.dev.device = p->dev.device; - set_irq_multimask.dev.function = p->dev.function; - set_irq_multimask.irq_type = p->irq_type; - - return esc_mods_set_irq_multimask(pfile, &set_irq_multimask); -} - int esc_mods_irq_handled_2(struct file *pfile, struct MODS_REGISTER_IRQ_2 *p) { - struct mods_priv *pmp = get_all_data(); - unsigned long flags = 0; - unsigned char channel; - u32 irq = p->dev.bus; + u8 client_id; + struct mods_client *client; + unsigned long flags = 0; + u32 irq = p->dev.bus; struct dev_irq_map *t = NULL; struct dev_irq_map *next = NULL; - int ret = -EINVAL; + int ret = -EINVAL; if (p->type != MODS_IRQ_TYPE_CPU) return -EINVAL; - /* Lock IRQ queue */ LOG_ENT(); - spin_lock_irqsave(&pmp->lock, flags); /* Identify the caller */ - channel = MODS_GET_FILE_PRIVATE_ID(pfile); - WARN_ON(id_is_valid(channel) != OK); - if (id_is_valid(channel) != OK) { + client_id = get_client_id(pfile); + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } + client = &mp.clients[client_id - 1]; /* Print info */ mods_debug_printk(DEBUG_ISR_DETAILED, "mark CPU IRQ 0x%x handled\n", irq); - list_for_each_entry_safe(t, next, &pmp->irq_head[channel-1], list) { + /* Lock IRQ queue */ + spin_lock_irqsave(&client->irq_lock, flags); + + list_for_each_entry_safe(t, next, &client->irq_list, list) { if (t->apic_irq == irq) { if (t->type != p->type) { mods_error_printk( @@ -1623,9 +1431,9 @@ int esc_mods_irq_handled_2(struct file *pfile, } /* Unlock IRQ queue */ - spin_unlock_irqrestore(&pmp->lock, flags); - LOG_EXT(); + spin_unlock_irqrestore(&client->irq_lock, flags); + LOG_EXT(); return ret; } diff --git a/drivers/misc/mods/mods_krnl.c b/drivers/misc/mods/mods_krnl.c index 7a5b39c6..21dfbd6a 100644 --- a/drivers/misc/mods/mods_krnl.c +++ b/drivers/misc/mods/mods_krnl.c @@ -126,7 +126,7 @@ struct pci_driver mods_pci_driver = { * used to avoid globalization of variables * ***********************************************/ -static int debug = -0x80000000; +static int debug; static int multi_instance = MODS_MULTI_INSTANCE_DEFAULT_VALUE; static u32 access_token = MODS_ACCESS_TOKEN_NONE; @@ -290,9 +290,9 @@ static int mods_set_access_token(u32 tok) static int mods_check_access_token(struct file *fp) { - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; - if (private_data->access_token != mods_get_access_token()) + if (client->access_token != mods_get_access_token()) return -EFAULT; return OK; @@ -307,6 +307,8 @@ static int __init mods_init_module(void) LOG_ENT(); + mods_init_irq(); + rc = misc_register(&mods_dev); if (rc < 0) return -EBUSY; @@ -317,8 +319,6 @@ static int __init mods_init_module(void) return -EBUSY; #endif - mods_init_irq(); - #if defined(MODS_HAS_CLOCK) mods_init_clock_api(); #endif @@ -402,26 +402,23 @@ MODULE_PARM_DESC(ppc_tce_bypass, /******************** * HELPER FUNCTIONS * ********************/ -static int id_is_valid(unsigned char channel) -{ - if (channel <= 0 || channel > MODS_CHANNEL_MAX) - return -EINVAL; - - return OK; -} - -static void mods_disable_all_devices(struct mods_file_private_data *priv) +static void mods_disable_all_devices(struct mods_client *client) { #ifdef CONFIG_PCI - while (priv->enabled_devices != 0) { - struct en_dev_entry *old = priv->enabled_devices; + if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex()))) + return; + + while (client->enabled_devices != 0) { + struct en_dev_entry *old = client->enabled_devices; mods_disable_device(old->dev); - priv->enabled_devices = old->next; + client->enabled_devices = old->next; kfree(old); } + + mutex_unlock(mods_get_irq_mutex()); #else - WARN_ON(priv->enabled_devices != 0); + WARN_ON(client->enabled_devices != 0); #endif } @@ -438,7 +435,7 @@ static int mods_register_mapping( u64 mapping_length) { struct SYS_MAP_MEMORY *p_map_mem; - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; LOG_ENT(); @@ -454,7 +451,7 @@ static int mods_register_mapping( p_map_mem->mapping_length = mapping_length; p_map_mem->p_mem_info = p_mem_info; - list_add(&p_map_mem->list, private_data->mods_mapping_list); + list_add(&p_map_mem->list, &client->mem_map_list); mods_debug_printk(DEBUG_MEM_DETAILED, "map alloc %p as %p: phys 0x%llx, virt 0x%llx, size 0x%llx\n", @@ -467,10 +464,9 @@ static int mods_register_mapping( static void mods_unregister_mapping(struct file *fp, u64 virtual_address) { struct SYS_MAP_MEMORY *p_map_mem; - MODS_PRIV private_data = fp->private_data; - - struct list_head *head = private_data->mods_mapping_list; - struct list_head *iter; + struct mods_client *client = fp->private_data; + struct list_head *head = &client->mem_map_list; + struct list_head *iter; LOG_ENT(); @@ -493,11 +489,10 @@ static void mods_unregister_mapping(struct file *fp, u64 virtual_address) static void mods_unregister_all_mappings(struct file *fp) { struct SYS_MAP_MEMORY *p_map_mem; - MODS_PRIV private_data = fp->private_data; - - struct list_head *head = private_data->mods_mapping_list; - struct list_head *iter; - struct list_head *tmp; + struct mods_client *client = fp->private_data; + struct list_head *head = &client->mem_map_list; + struct list_head *iter; + struct list_head *tmp; LOG_ENT(); @@ -531,12 +526,12 @@ static pgprot_t mods_get_prot(u32 mem_type, pgprot_t prot) static pgprot_t mods_get_prot_for_range(struct file *fp, u64 dma_addr, u64 size, pgprot_t prot) { - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; - if ((dma_addr == private_data->mem_type.dma_addr) && - (size == private_data->mem_type.size)) { + if ((dma_addr == client->mem_type.dma_addr) && + (size == client->mem_type.size)) { - return mods_get_prot(private_data->mem_type.type, prot); + return mods_get_prot(client->mem_type.type, prot); } return prot; } @@ -562,12 +557,12 @@ static const char *mods_get_prot_str_for_range(struct file *fp, u64 dma_addr, u64 size) { - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; - if ((dma_addr == private_data->mem_type.dma_addr) && - (size == private_data->mem_type.size)) { + if ((dma_addr == client->mem_type.dma_addr) && + (size == client->mem_type.size)) { - return mods_get_prot_str(private_data->mem_type.type); + return mods_get_prot_str(client->mem_type.type); } return "default"; } @@ -639,11 +634,11 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma) struct mods_vm_private_data *vma_private_data = MODS_VMA_PRIVATE(vma); if (atomic_dec_and_test(&vma_private_data->usage_count)) { - MODS_PRIV private_data = + struct mods_client *client = vma_private_data->fp->private_data; if (unlikely(mutex_lock_interruptible( - &private_data->mtx))) { + &client->mtx))) { LOG_EXT(); return; } @@ -657,7 +652,7 @@ static void mods_krnl_vma_close(struct vm_area_struct *vma) MODS_VMA_PRIVATE(vma) = NULL; kfree(vma_private_data); - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); } } LOG_EXT(); @@ -670,126 +665,18 @@ static const struct vm_operations_struct mods_krnl_vm_ops = { static int mods_krnl_open(struct inode *ip, struct file *fp) { - struct list_head *mods_alloc_list; - struct list_head *mods_mapping_list; - struct list_head *mods_pci_res_map_list; -#if defined(CONFIG_PPC64) - struct list_head *mods_ppc_tce_bypass_list; - struct list_head *mods_nvlink_sysmem_trained_list; -#endif - struct mods_file_private_data *private_data; - int id = 0; - int i = 0; + struct mods_client *client; LOG_ENT(); - mods_alloc_list = kmalloc(sizeof(struct list_head), - GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!mods_alloc_list)) { - LOG_EXT(); - return -ENOMEM; - } - - mods_mapping_list = kmalloc(sizeof(struct list_head), - GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!mods_mapping_list)) { - kfree(mods_alloc_list); - LOG_EXT(); - return -ENOMEM; - } - - mods_pci_res_map_list = kmalloc(sizeof(struct list_head), - GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!mods_pci_res_map_list)) { - kfree(mods_alloc_list); - kfree(mods_mapping_list); - LOG_EXT(); - return -ENOMEM; - } - -#if defined(CONFIG_PPC64) - mods_ppc_tce_bypass_list = - kmalloc(sizeof(struct list_head), GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!mods_ppc_tce_bypass_list)) { - kfree(mods_alloc_list); - kfree(mods_mapping_list); - kfree(mods_pci_res_map_list); - LOG_EXT(); - return -ENOMEM; - } - - mods_nvlink_sysmem_trained_list = - kmalloc(sizeof(struct list_head), GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!mods_nvlink_sysmem_trained_list)) { - kfree(mods_alloc_list); - kfree(mods_mapping_list); - kfree(mods_pci_res_map_list); - kfree(mods_ppc_tce_bypass_list); - LOG_EXT(); - return -ENOMEM; - } -#endif - - private_data = kmalloc(sizeof(*private_data), - GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!private_data)) { - kfree(mods_alloc_list); - kfree(mods_mapping_list); - kfree(mods_pci_res_map_list); -#if defined(CONFIG_PPC64) - kfree(mods_ppc_tce_bypass_list); - kfree(mods_nvlink_sysmem_trained_list); -#endif - LOG_EXT(); - return -ENOMEM; - } - - id = mods_alloc_channel(); - if (id_is_valid(id) != OK) { + client = mods_alloc_client(); + if (client == NULL) { mods_error_printk("too many clients\n"); - kfree(mods_alloc_list); - kfree(mods_mapping_list); - kfree(mods_pci_res_map_list); -#if defined(CONFIG_PPC64) - kfree(mods_ppc_tce_bypass_list); - kfree(mods_nvlink_sysmem_trained_list); -#endif - kfree(private_data); LOG_EXT(); return -EBUSY; } - private_data->mods_id = id; - mods_irq_dev_set_pri(id, private_data); - - INIT_LIST_HEAD(mods_alloc_list); - INIT_LIST_HEAD(mods_mapping_list); - INIT_LIST_HEAD(mods_pci_res_map_list); - private_data->mods_alloc_list = mods_alloc_list; - private_data->mods_mapping_list = mods_mapping_list; - private_data->mods_pci_res_map_list = mods_pci_res_map_list; -#if defined(CONFIG_PPC64) - INIT_LIST_HEAD(mods_ppc_tce_bypass_list); - INIT_LIST_HEAD(mods_nvlink_sysmem_trained_list); - private_data->mods_ppc_tce_bypass_list = mods_ppc_tce_bypass_list; - private_data->mods_nvlink_sysmem_trained_list - = mods_nvlink_sysmem_trained_list; -#endif - private_data->enabled_devices = 0; - private_data->mem_type.dma_addr = 0; - private_data->mem_type.size = 0; - private_data->mem_type.type = 0; - - mutex_init(&private_data->mtx); - - for (i = 0; i < FB_MAX; i++) - private_data->mods_fb_suspended[i] = 0; - - init_waitqueue_head(&private_data->interrupt_event); - - private_data->access_token = MODS_ACCESS_TOKEN_NONE; - - fp->private_data = private_data; + fp->private_data = client; mods_info_printk("driver opened\n"); LOG_EXT(); @@ -798,30 +685,26 @@ static int mods_krnl_open(struct inode *ip, struct file *fp) static int mods_krnl_close(struct inode *ip, struct file *fp) { - MODS_PRIV private_data = fp->private_data; - unsigned char id = MODS_GET_FILE_PRIVATE_ID(fp); - int ret = OK; + struct mods_client *client = fp->private_data; + u8 client_id = client->client_id; + int ret = OK; LOG_ENT(); - WARN_ON(id_is_valid(id) != OK); - if (id_is_valid(id) != OK) { + WARN_ON(!is_client_id_valid(client_id)); + if (!is_client_id_valid(client_id)) { LOG_EXT(); return -EINVAL; } - mods_resume_console(fp); + mods_free_client_interrupts(client); - mods_free_channel(id); - mods_irq_dev_clr_pri(id); + mods_resume_console(fp); mods_unregister_all_mappings(fp); ret = mods_unregister_all_alloc(fp); if (ret) mods_error_printk("failed to free all memory\n"); - ret = MODS_UNREGISTER_PCI_MAP(fp); - if (ret) - mods_error_printk("failed to free pci mappings\n"); #if defined(CONFIG_PPC64) ret = mods_unregister_all_ppc_tce_bypass(fp); @@ -833,16 +716,9 @@ static int mods_krnl_close(struct inode *ip, struct file *fp) mods_error_printk("failed to free nvlink trained\n"); #endif - mods_disable_all_devices(private_data); + mods_disable_all_devices(client); - kfree(private_data->mods_alloc_list); - kfree(private_data->mods_mapping_list); - kfree(private_data->mods_pci_res_map_list); -#if defined(CONFIG_PPC64) - kfree(private_data->mods_ppc_tce_bypass_list); - kfree(private_data->mods_nvlink_sysmem_trained_list); -#endif - kfree(private_data); + mods_free_client(client_id); mods_info_printk("driver closed\n"); LOG_EXT(); @@ -852,8 +728,8 @@ static int mods_krnl_close(struct inode *ip, struct file *fp) static unsigned int mods_krnl_poll(struct file *fp, poll_table *wait) { unsigned int mask = 0; - MODS_PRIV private_data = fp->private_data; - unsigned char id = MODS_GET_FILE_PRIVATE_ID(fp); + struct mods_client *client = fp->private_data; + u8 client_id = get_client_id(fp); int access_tok_ret = mods_check_access_token(fp); if (access_tok_ret < 0) @@ -861,10 +737,10 @@ static unsigned int mods_krnl_poll(struct file *fp, poll_table *wait) if (!(fp->f_flags & O_NONBLOCK)) { mods_debug_printk(DEBUG_ISR_DETAILED, "poll wait\n"); - poll_wait(fp, &private_data->interrupt_event, wait); + poll_wait(fp, &client->interrupt_event, wait); } /* if any interrupts pending then check intr, POLLIN on irq */ - mask |= mods_irq_event_check(id); + mask |= mods_irq_event_check(client_id); mods_debug_printk(DEBUG_ISR_DETAILED, "poll mask 0x%x\n", mask); return mask; } @@ -903,13 +779,13 @@ static int mods_krnl_mmap(struct file *fp, struct vm_area_struct *vma) { int ret = OK; - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) + if (unlikely(mutex_lock_interruptible(&client->mtx))) ret = -EINTR; else { ret = mods_krnl_map_inner(fp, vma); - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); } LOG_EXT(); return ret; @@ -1162,14 +1038,14 @@ static int esc_mods_suspend_console(struct file *pfile) #if defined(CONFIG_FB) && defined(MODS_HAS_CONSOLE_LOCK) if (num_registered_fb) { /* tell the os to block fb accesses */ - MODS_PRIV private_data = pfile->private_data; + struct mods_client *client = pfile->private_data; int i = 0; for (i = 0; i < num_registered_fb; i++) { console_lock(); if (registered_fb[i]->state != FBINFO_STATE_SUSPENDED) { fb_set_suspend(registered_fb[i], 1); - private_data->mods_fb_suspended[i] = 1; + client->mods_fb_suspended[i] = 1; } console_unlock(); } @@ -1207,14 +1083,14 @@ static int mods_resume_console(struct file *pfile) #if defined(CONFIG_FB) && defined(MODS_HAS_CONSOLE_LOCK) if (num_registered_fb) { - MODS_PRIV private_data = pfile->private_data; + struct mods_client *client = pfile->private_data; int i = 0; for (i = 0; i < num_registered_fb; i++) { console_lock(); - if (private_data->mods_fb_suspended[i]) { + if (client->mods_fb_suspended[i]) { fb_set_suspend(registered_fb[i], 0); - private_data->mods_fb_suspended[i] = 0; + client->mods_fb_suspended[i] = 0; } console_unlock(); } @@ -1258,9 +1134,9 @@ static int esc_mods_acquire_access_token(struct file *pfile, if (ret < 0) { mods_error_printk("unable to set access token!\n"); } else { - MODS_PRIV private_data = pfile->private_data; + struct mods_client *client = pfile->private_data; - private_data->access_token = ptoken->token; + client->access_token = ptoken->token; } LOG_EXT(); @@ -1286,9 +1162,9 @@ static int esc_mods_release_access_token(struct file *pfile, if (ret < 0) { mods_error_printk("unable to clear access token!\n"); } else { - MODS_PRIV private_data = pfile->private_data; + struct mods_client *client = pfile->private_data; - private_data->access_token = MODS_ACCESS_TOKEN_NONE; + client->access_token = MODS_ACCESS_TOKEN_NONE; } LOG_EXT(); @@ -1304,9 +1180,9 @@ static int esc_mods_verify_access_token(struct file *pfile, LOG_ENT(); if (ptoken->token == mods_get_access_token()) { - MODS_PRIV private_data = pfile->private_data; + struct mods_client *client = pfile->private_data; - private_data->access_token = ptoken->token; + client->access_token = ptoken->token; ret = OK; } else mods_error_printk("invalid access token\n"); @@ -1596,22 +1472,23 @@ static long mods_krnl_ioctl(struct file *fp, MODS_DEVICE_NUMA_INFO_2); break; - case MODS_ESC_PCI_MAP_RESOURCE: - MODS_IOCTL(MODS_ESC_PCI_MAP_RESOURCE, - esc_mods_pci_map_resource, - MODS_PCI_MAP_RESOURCE); - break; - - case MODS_ESC_PCI_UNMAP_RESOURCE: - MODS_IOCTL(MODS_ESC_PCI_UNMAP_RESOURCE, - esc_mods_pci_unmap_resource, - MODS_PCI_UNMAP_RESOURCE); - break; case MODS_ESC_GET_IOMMU_STATE: MODS_IOCTL(MODS_ESC_GET_IOMMU_STATE, esc_mods_get_iommu_state, MODS_GET_IOMMU_STATE); break; + + case MODS_ESC_GET_IOMMU_STATE_2: + MODS_IOCTL(MODS_ESC_GET_IOMMU_STATE_2, + esc_mods_get_iommu_state_2, + MODS_GET_IOMMU_STATE); + break; + + case MODS_ESC_PCI_SET_DMA_MASK: + MODS_IOCTL(MODS_ESC_PCI_SET_DMA_MASK, + esc_mods_pci_set_dma_mask, + MODS_PCI_DMA_MASK); + break; #endif case MODS_ESC_ALLOC_PAGES: @@ -1767,23 +1644,6 @@ static long mods_krnl_ioctl(struct file *fp, esc_mods_query_irq_2, MODS_QUERY_IRQ_2); break; - case MODS_ESC_SET_IRQ_MASK: - MODS_IOCTL_NORETVAL(MODS_ESC_SET_IRQ_MASK, - esc_mods_set_irq_mask, MODS_SET_IRQ_MASK); - break; - - case MODS_ESC_SET_IRQ_MASK_2: - MODS_IOCTL_NORETVAL(MODS_ESC_SET_IRQ_MASK_2, - esc_mods_set_irq_mask_2, - MODS_SET_IRQ_MASK_2); - break; - - case MODS_ESC_SET_IRQ_MULTIMASK: - MODS_IOCTL_NORETVAL(MODS_ESC_SET_IRQ_MULTIMASK, - esc_mods_set_irq_multimask, - MODS_SET_IRQ_MULTIMASK); - break; - case MODS_ESC_IRQ_HANDLED: MODS_IOCTL_NORETVAL(MODS_ESC_IRQ_HANDLED, esc_mods_irq_handled, MODS_REGISTER_IRQ); diff --git a/drivers/misc/mods/mods_mem.c b/drivers/misc/mods/mods_mem.c index b3f606b1..09c96d88 100644 --- a/drivers/misc/mods/mods_mem.c +++ b/drivers/misc/mods/mods_mem.c @@ -77,7 +77,7 @@ static int mods_dma_unmap_and_free(struct MODS_MEM_INFO *p_mem_info, struct list_head *head; struct list_head *iter; - head = p_mem_info->dma_map_list; + head = &p_mem_info->dma_map_list; list_for_each(iter, head) { p_dma_map = list_entry(iter, struct MODS_DMA_MAP, list); @@ -115,13 +115,10 @@ static int mods_dma_unmap_and_free(struct MODS_MEM_INFO *p_mem_info, int mods_dma_unmap_all(struct MODS_MEM_INFO *p_mem_info, struct pci_dev *p_pci_dev) { - struct list_head *head = p_mem_info->dma_map_list; + struct list_head *head = &p_mem_info->dma_map_list; struct list_head *iter; struct list_head *tmp; - if (!p_mem_info->dma_map_list) - return OK; - list_for_each_safe(iter, tmp, head) { struct MODS_DMA_MAP *p_dma_map; int ret; @@ -182,17 +179,7 @@ static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, struct pci_dev *p_pci_dev) { struct MODS_DMA_MAP *p_dma_map; - int list_allocated = 0; - u32 alloc_size; - - if (!p_mem_info->dma_map_list) { - p_mem_info->dma_map_list = kmalloc(sizeof(struct list_head), - GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!p_mem_info->dma_map_list)) - return -ENOMEM; - INIT_LIST_HEAD(p_mem_info->dma_map_list); - list_allocated = 1; - } + u32 alloc_size; alloc_size = sizeof(*p_dma_map) + (p_mem_info->max_chunks - 1) * @@ -201,18 +188,14 @@ static int mods_create_dma_map(struct MODS_MEM_INFO *p_mem_info, p_dma_map = kmalloc(alloc_size, GFP_KERNEL | __GFP_NORETRY); if (unlikely(!p_dma_map)) { mods_error_printk("failed to allocate device map data\n"); - if (list_allocated) { - kfree(p_mem_info->dma_map_list); - p_mem_info->dma_map_list = NULL; - return -ENOMEM; - } + return -ENOMEM; } memset(p_dma_map, 0, alloc_size); p_dma_map->dev = p_pci_dev; mods_dma_map_pages(p_mem_info, p_dma_map); - list_add(&p_dma_map->list, p_mem_info->dma_map_list); + list_add(&p_dma_map->list, &p_mem_info->dma_map_list); return OK; } @@ -230,7 +213,7 @@ static struct MODS_MAP_CHUNK *mods_find_dma_map_chunk( struct list_head *iter; int i; - head = p_mem_info->dma_map_list; + head = &p_mem_info->dma_map_list; if (!head) return NULL; @@ -485,12 +468,12 @@ failed: static int mods_register_alloc(struct file *fp, struct MODS_MEM_INFO *p_mem_info) { - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) + if (unlikely(mutex_lock_interruptible(&client->mtx))) return -EINTR; - list_add(&p_mem_info->list, private_data->mods_alloc_list); - mutex_unlock(&private_data->mtx); + list_add(&p_mem_info->list, &client->mem_alloc_list); + mutex_unlock(&client->mtx); return OK; } @@ -498,16 +481,16 @@ static int mods_unregister_and_free(struct file *fp, struct MODS_MEM_INFO *p_del_mem) { struct MODS_MEM_INFO *p_mem_info; - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; struct list_head *head; struct list_head *iter; mods_debug_printk(DEBUG_MEM_DETAILED, "free %p\n", p_del_mem); - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) + if (unlikely(mutex_lock_interruptible(&client->mtx))) return -EINTR; - head = private_data->mods_alloc_list; + head = &client->mem_alloc_list; list_for_each(iter, head) { p_mem_info = list_entry(iter, struct MODS_MEM_INFO, list); @@ -515,7 +498,7 @@ static int mods_unregister_and_free(struct file *fp, if (p_del_mem == p_mem_info) { list_del(iter); - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_dma_unmap_all(p_mem_info, NULL); mods_restore_cache(p_mem_info); @@ -527,7 +510,7 @@ static int mods_unregister_and_free(struct file *fp, } } - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_error_printk("failed to unregister allocation %p\n", p_del_mem); @@ -536,10 +519,10 @@ static int mods_unregister_and_free(struct file *fp, int mods_unregister_all_alloc(struct file *fp) { - MODS_PRIV private_data = fp->private_data; - struct list_head *head = private_data->mods_alloc_list; - struct list_head *iter; - struct list_head *tmp; + struct mods_client *client = fp->private_data; + struct list_head *head = &client->mem_alloc_list; + struct list_head *iter; + struct list_head *tmp; list_for_each_safe(iter, tmp, head) { struct MODS_MEM_INFO *p_mem_info; @@ -587,8 +570,8 @@ int mods_get_alloc_offset(struct MODS_MEM_INFO *p_mem_info, struct MODS_MEM_INFO *mods_find_alloc(struct file *fp, u64 phys_addr) { - MODS_PRIV private_data = fp->private_data; - struct list_head *plist_head = private_data->mods_alloc_list; + struct mods_client *client = fp->private_data; + struct list_head *plist_head = &client->mem_alloc_list; struct list_head *plist_iter; struct MODS_MEM_INFO *p_mem_info; u64 offset; @@ -732,9 +715,10 @@ int esc_mods_device_alloc_pages_2(struct file *fp, p_mem_info->addr_bits = p->address_bits; p_mem_info->num_pages = num_pages; p_mem_info->numa_node = numa_node_id(); - p_mem_info->dma_map_list = NULL; p_mem_info->dev = NULL; + INIT_LIST_HEAD(&p_mem_info->dma_map_list); + if (p->pci_device.bus || p->pci_device.device) { unsigned int devfn = PCI_DEVFN(p->pci_device.device, p->pci_device.function); @@ -799,13 +783,11 @@ int esc_mods_device_alloc_pages_2(struct file *fp, mods_debug_printk(DEBUG_MEM_DETAILED, "alloc %p\n", p_mem_info); ret = mods_register_alloc(fp, p_mem_info); - LOG_EXT(); - return ret; + failed: - if (p_mem_info) { - kfree(p_mem_info->dma_map_list); + if (ret) kfree(p_mem_info); - } + LOG_EXT(); return ret; } @@ -876,7 +858,7 @@ int esc_mods_free_pages(struct file *fp, struct MODS_FREE_PAGES *p) int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) { struct MODS_MEM_INFO *p_mem_info; - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; LOG_ENT(); @@ -892,25 +874,25 @@ int esc_mods_set_mem_type(struct file *fp, struct MODS_MEMORY_TYPE *p) return -EINVAL; } - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) { + if (unlikely(mutex_lock_interruptible(&client->mtx))) { LOG_EXT(); return -EINTR; } p_mem_info = mods_find_alloc(fp, p->physical_address); if (p_mem_info) { - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_error_printk("cannot set mem type on phys addr 0x%llx\n", p->physical_address); LOG_EXT(); return -EINVAL; } - private_data->mem_type.dma_addr = p->physical_address; - private_data->mem_type.size = p->size; - private_data->mem_type.type = p->type; + client->mem_type.dma_addr = p->physical_address; + client->mem_type.size = p->size; + client->mem_type.type = p->type; - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); LOG_EXT(); return OK; @@ -1071,18 +1053,18 @@ int esc_mods_virtual_to_phys(struct file *fp, struct MODS_VIRTUAL_TO_PHYSICAL *p) { struct MODS_GET_PHYSICAL_ADDRESS get_phys_addr; - MODS_PRIV private_data = fp->private_data; - struct list_head *head; - struct list_head *iter; + struct mods_client *client = fp->private_data; + struct list_head *head; + struct list_head *iter; LOG_ENT(); - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) { + if (unlikely(mutex_lock_interruptible(&client->mtx))) { LOG_EXT(); return -EINTR; } - head = private_data->mods_mapping_list; + head = &client->mem_map_list; list_for_each(iter, head) { struct SYS_MAP_MEMORY *p_map_mem; @@ -1103,7 +1085,7 @@ int esc_mods_virtual_to_phys(struct file *fp, if (!p_map_mem->p_mem_info) { p->physical_address = p_map_mem->dma_addr + virt_offs; - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_debug_printk(DEBUG_MEM_DETAILED, "get phys: map %p virt 0x%llx -> 0x%llx\n", @@ -1123,7 +1105,7 @@ int esc_mods_virtual_to_phys(struct file *fp, (u64)(size_t)p_map_mem->p_mem_info; get_phys_addr.offset = virt_offs + phys_offs; - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); ret = esc_mods_get_phys_addr(fp, &get_phys_addr); if (ret != OK) @@ -1140,7 +1122,7 @@ int esc_mods_virtual_to_phys(struct file *fp, } } - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_error_printk("invalid virtual address\n"); return -EINVAL; @@ -1150,7 +1132,7 @@ int esc_mods_phys_to_virtual(struct file *fp, struct MODS_PHYSICAL_TO_VIRTUAL *p) { struct SYS_MAP_MEMORY *p_map_mem; - MODS_PRIV private_data = fp->private_data; + struct mods_client *client = fp->private_data; struct list_head *head; struct list_head *iter; u64 offset; @@ -1158,12 +1140,12 @@ int esc_mods_phys_to_virtual(struct file *fp, LOG_ENT(); - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) { + if (unlikely(mutex_lock_interruptible(&client->mtx))) { LOG_EXT(); return -EINTR; } - head = private_data->mods_mapping_list; + head = &client->mem_map_list; list_for_each(iter, head) { p_map_mem = list_entry(iter, struct SYS_MAP_MEMORY, list); @@ -1180,7 +1162,7 @@ int esc_mods_phys_to_virtual(struct file *fp, - p_map_mem->dma_addr; p->virtual_address = p_map_mem->virtual_addr + offset; - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_debug_printk(DEBUG_MEM_DETAILED, "get virt: map %p phys 0x%llx -> 0x%llx\n", @@ -1207,7 +1189,7 @@ int esc_mods_phys_to_virtual(struct file *fp, p->virtual_address = p_map_mem->virtual_addr + offset - map_offset; - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_debug_printk(DEBUG_MEM_DETAILED, "get virt: map %p phys 0x%llx -> 0x%llx\n", p_map_mem, p->physical_address, p->virtual_address); @@ -1216,7 +1198,7 @@ int esc_mods_phys_to_virtual(struct file *fp, return OK; } } - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); mods_error_printk("phys addr 0x%llx is not mapped\n", p->physical_address); return -EINVAL; @@ -1402,9 +1384,9 @@ static void clear_entry_cache_mappings int esc_mods_flush_cpu_cache_range(struct file *fp, struct MODS_FLUSH_CPU_CACHE_RANGE *p) { - MODS_PRIV private_data = fp->private_data; - struct list_head *head; - struct list_head *iter; + struct mods_client *client = fp->private_data; + struct list_head *head; + struct list_head *iter; if (irqs_disabled() || in_interrupt() || p->virt_addr_start > p->virt_addr_end || @@ -1414,12 +1396,12 @@ int esc_mods_flush_cpu_cache_range(struct file *fp, return -EINVAL; } - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) { + if (unlikely(mutex_lock_interruptible(&client->mtx))) { LOG_EXT(); return -EINTR; } - head = private_data->mods_mapping_list; + head = &client->mem_map_list; list_for_each(iter, head) { struct SYS_MAP_MEMORY *p_map_mem @@ -1455,7 +1437,7 @@ int esc_mods_flush_cpu_cache_range(struct file *fp, virt_end); } } - mutex_unlock(&private_data->mtx); + mutex_unlock(&client->mtx); return OK; } diff --git a/drivers/misc/mods/mods_pci.c b/drivers/misc/mods/mods_pci.c index e5cdfb05..2c701655 100644 --- a/drivers/misc/mods/mods_pci.c +++ b/drivers/misc/mods/mods_pci.c @@ -25,84 +25,6 @@ #include #endif -/************************ - * PCI HELPER FUNCTIONS * - ************************/ - -static int mods_free_pci_res_map(struct file *fp, - struct MODS_PCI_RES_MAP_INFO *p_del_map) -{ -#if defined(MODS_HAS_PCI_MAP_RESOURCE) - struct MODS_PCI_RES_MAP_INFO *p_res_map; - MODS_PRIV private_data = fp->private_data; - struct list_head *head; - struct list_head *iter; - - mods_debug_printk(DEBUG_PCI, - "free pci resource map %p\n", - p_del_map); - - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) - return -EINTR; - - head = private_data->mods_pci_res_map_list; - - list_for_each(iter, head) { - p_res_map = - list_entry(iter, struct MODS_PCI_RES_MAP_INFO, list); - - if (p_del_map == p_res_map) { - list_del(iter); - - mutex_unlock(&private_data->mtx); - - pci_unmap_resource(p_res_map->dev, - p_res_map->va, - p_res_map->page_count * PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - mods_debug_printk(DEBUG_PCI, - "unmapped pci resource at 0x%llx from %u:%u:%u.%u\n", - p_res_map->va, - pci_domain_nr(p_res_map->dev->bus), - p_res_map->dev->bus->number, - PCI_SLOT(p_res_map->dev->devfn), - PCI_FUNC(p_res_map->dev->devfn)); - kfree(p_res_map); - return OK; - } - } - - mutex_unlock(&private_data->mtx); - - mods_error_printk("failed to unregister pci resource mapping %p\n", - p_del_map); - return -EINVAL; -#else - return OK; -#endif -} - -int mods_unregister_all_pci_res_mappings(struct file *fp) -{ - MODS_PRIV private_data = fp->private_data; - struct list_head *head = private_data->mods_pci_res_map_list; - struct list_head *iter; - struct list_head *tmp; - - list_for_each_safe(iter, tmp, head) { - struct MODS_PCI_RES_MAP_INFO *p_pci_res_map_info; - int ret; - - p_pci_res_map_info = - list_entry(iter, struct MODS_PCI_RES_MAP_INFO, list); - ret = mods_free_pci_res_map(fp, p_pci_res_map_info); - if (ret) - return ret; - } - - return OK; -} - /************************ * PCI ESCAPE FUNCTIONS * ************************/ @@ -134,19 +56,6 @@ static int mods_find_pci_dev(struct file *pfile, p->pci_device.device = PCI_SLOT(dev->devfn); p->pci_device.function = PCI_FUNC(dev->devfn); -#if defined(CONFIG_PPC64) - /* Enable device on the PCI bus */ - if (mods_enable_device(pfile->private_data, dev)) { - mods_error_printk( - "unable to enable dev %04x:%02x:%02x.%x\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function); - return -EINVAL; - } -#endif - return OK; } @@ -201,19 +110,6 @@ static int mods_find_pci_class_code(struct file *pfile, p->pci_device.device = PCI_SLOT(dev->devfn); p->pci_device.function = PCI_FUNC(dev->devfn); -#if defined(CONFIG_PPC64) - /* Enable device on the PCI bus */ - if (mods_enable_device(pfile->private_data, dev)) { - mods_error_printk( - "unable to enable dev %04x:%02x:%02x.%x\n", - (unsigned int)p->pci_device.domain, - (unsigned int)p->pci_device.bus, - (unsigned int)p->pci_device.device, - (unsigned int)p->pci_device.function); - return -EINVAL; - } -#endif - return OK; } @@ -264,6 +160,27 @@ int esc_mods_pci_get_bar_info_2(struct file *pfile, (int) p->pci_device.bus, (int) p->pci_device.device, (int) p->pci_device.function, (int) p->bar_index); +#if defined(CONFIG_PPC64) + if (unlikely(mutex_lock_interruptible(mods_get_irq_mutex()))) { + LOG_EXT(); + return -EINTR; + } + + /* Enable device on the PCI bus */ + if (mods_enable_device(pfile->private_data, dev) == 0) { + mods_error_printk( + "unable to enable dev %04x:%02x:%02x.%x\n", + (unsigned int)p->pci_device.domain, + (unsigned int)p->pci_device.bus, + (unsigned int)p->pci_device.device, + (unsigned int)p->pci_device.function); + mutex_unlock(mods_get_irq_mutex()); + return -EINVAL; + } + + mutex_unlock(mods_get_irq_mutex()); +#endif + bar_resource_offset = 0; for (i = 0; i < p->bar_index; i++) { #if defined(MODS_HAS_IORESOURCE_MEM_64) @@ -665,188 +582,22 @@ int esc_mods_device_numa_info(struct file *fp, return OK; } -int esc_mods_pci_map_resource(struct file *fp, - struct MODS_PCI_MAP_RESOURCE *p) -{ -#if defined(MODS_HAS_PCI_MAP_RESOURCE) - MODS_PRIV private_data = fp->private_data; - unsigned int devfn; - struct pci_dev *rem_dev; - struct pci_dev *loc_dev; - struct MODS_PCI_RES_MAP_INFO *p_res_map; - - LOG_ENT(); - - devfn = PCI_DEVFN(p->local_pci_device.device, - p->local_pci_device.function); - loc_dev = MODS_PCI_GET_SLOT(p->local_pci_device.domain, - p->local_pci_device.bus, devfn); - if (!loc_dev) { - mods_error_printk("Local PCI device %04x:%x:%02x.%x not found\n", - p->local_pci_device.domain, - p->local_pci_device.bus, - p->local_pci_device.device, - p->local_pci_device.function); - LOG_EXT(); - return -EINVAL; - } - - devfn = PCI_DEVFN(p->remote_pci_device.device, - p->remote_pci_device.function); - rem_dev = MODS_PCI_GET_SLOT(p->remote_pci_device.domain, - p->remote_pci_device.bus, devfn); - if (!rem_dev) { - mods_error_printk("Remote PCI device %04x:%x:%02x.%x not found\n", - p->remote_pci_device.domain, - p->remote_pci_device.bus, - p->remote_pci_device.device, - p->remote_pci_device.function); - LOG_EXT(); - return -EINVAL; - } - - if ((p->resource_index >= DEVICE_COUNT_RESOURCE) || - !pci_resource_len(rem_dev, p->resource_index)) { - mods_error_printk( - "Resource %u on device %04x:%x:%02x.%x not found\n", - p->resource_index, - p->remote_pci_device.domain, - p->remote_pci_device.bus, - p->remote_pci_device.device, - p->remote_pci_device.function); - LOG_EXT(); - return -EINVAL; - } - - if ((p->va < pci_resource_start(rem_dev, p->resource_index)) || - (p->va > pci_resource_end(rem_dev, p->resource_index)) || - (p->va + p->page_count * PAGE_SIZE > - pci_resource_end(rem_dev, p->resource_index))) { - mods_error_printk( - "bad resource address 0x%04x:%x:%02x.%x on device %u:%u:%u.%u\n" - (unsigned long long)p->va, - p->remote_pci_device.domain, - p->remote_pci_device.bus, - p->remote_pci_device.device, - p->remote_pci_device.function); - LOG_EXT(); - return -EINVAL; - } - - p_res_map = kmalloc(sizeof(struct MODS_PCI_RES_MAP_INFO), - GFP_KERNEL | __GFP_NORETRY); - if (unlikely(!p_res_map)) { - mods_error_printk("failed to allocate pci res map struct\n"); - LOG_EXT(); - return -ENOMEM; - } - - p_res_map->dev = loc_dev; - p_res_map->page_count = p->page_count; - p_res_map->va = pci_map_resource(loc_dev, - &rem_dev->resource[resource_index], - p->va - pci_resource_start(rem_dev, p->resource_index), - p->page_count * PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - p_res_map->va = p->va; - if (pci_dma_mapping_error(loc_dev, p_res_map->va)) { - kfree(p_res_map); - LOG_EXT(); - return -ENOMEM; - } - - if (unlikely(mutex_lock_interruptible(&private_data->mtx))) { - kfree(p_res_map); - LOG_EXT(); - return -EINTR; - } - list_add(&p_res_map->list, private_data->mods_pci_res_map_list); - mutex_unlock(&private_data->mtx); - - p->va = p_res_map->va; - - mods_debug_printk(DEBUG_PCI, - "mapped pci resource %u from %u:%u:%u.%u to %u:%u:%u.%u at 0x%llx\n", - p->resource_index, - p->remote_pci_device.domain, - p->remote_pci_device.bus, - p->remote_pci_device.device, - p->remote_pci_device.function, - p->local_pci_device.domain, - p->local_pci_device.bus, - p->local_pci_device.device, - p->local_pci_device.function, - p->va); -#else - /* - * We still return OK, in case the system is running an older kernel - * with the IOMMU disabled. The va parameter will still contain the - * input physical address, which is what the device should use in this - * fallback case. - */ -#endif - return OK; -} - -int esc_mods_pci_unmap_resource(struct file *fp, - struct MODS_PCI_UNMAP_RESOURCE *p) -{ -#if defined(MODS_HAS_PCI_MAP_RESOURCE) - MODS_PRIV private_data = fp->private_data; - unsigned int devfn = PCI_DEVFN(p->pci_device.device, - p->pci_device.function); - struct pci_dev *dev = MODS_PCI_GET_SLOT(p->pci_device.domain, - p->pci_device.bus, devfn); - struct list_head *head = private_data->mods_pci_res_map_list; - struct list_head *iter; - struct list_head *tmp; - - LOG_ENT(); - - if (!dev) { - mods_error_printk("PCI device %04x:%x:%02x.%x not found\n", - p->pci_device.domain, - p->pci_device.bus, - p->pci_device.device, - p->pci_device.function); - LOG_EXT(); - return -EINVAL; - } - - list_for_each_safe(iter, tmp, head) { - struct MODS_PCI_RES_MAP_INFO *p_pci_res_map_info; - - p_pci_res_map_info = - list_entry(iter, struct MODS_PCI_RES_MAP_INFO, list); - - if ((p_pci_res_map_info->dev == dev) && - (p_pci_res_map_info->va == p->va)) { - int ret = mods_free_pci_res_map(fp, p_pci_res_map_info); - - LOG_EXT(); - return ret; - } - } - - mods_error_printk( - "PCI mapping 0x%llx on device %04x:%x:%02x.%x not found\n", - p->va, - p->pci_device.domain, - p->pci_device.bus, - p->pci_device.device, - p->pci_device.function); - return -EINVAL; -#else - return OK; -#endif -} - int esc_mods_get_iommu_state(struct file *pfile, struct MODS_GET_IOMMU_STATE *state) +{ + int err = esc_mods_get_iommu_state_2(pfile, state); + + if (!err) + state->state = (state->state == MODS_SWIOTLB_DISABLED) ? 1 : 0; + + return err; +} + +int esc_mods_get_iommu_state_2(struct file *pfile, + struct MODS_GET_IOMMU_STATE *state) { #if !defined(CONFIG_SWIOTLB) - /* SWIOTLB turned off in the kernel, HW IOMMU active */ - state->state = 1; + state->state = MODS_SWIOTLB_DISABLED; #elif defined(MODS_HAS_DMA_OPS) && \ (defined(MODS_HAS_NONCOH_DMA_OPS) || defined(MODS_HAS_MAP_SG_ATTRS)) @@ -859,18 +610,53 @@ int esc_mods_get_iommu_state(struct file *pfile, const struct dma_map_ops *ops = get_dma_ops(&dev->dev); #if defined(MODS_HAS_NONCOH_DMA_OPS) - state->state = ops != &noncoherent_swiotlb_dma_ops && - ops != &coherent_swiotlb_dma_ops; + state->state = (ops != &noncoherent_swiotlb_dma_ops && + ops != &coherent_swiotlb_dma_ops) + ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE; #else - state->state = ops->map_sg != swiotlb_map_sg_attrs; + state->state = ops->map_sg != swiotlb_map_sg_attrs + ? MODS_SWIOTLB_DISABLED : MODS_SWIOTLB_ACTIVE; #endif #elif defined(CONFIG_PPC64) || defined(CONFIG_ARM64) - /* Old/new kernels, no way to detect, default to HW IOMMU active */ - state->state = 1; + /* No way to detect, assume SW I/O TLB is disabled on ppc64/arm64 */ + state->state = MODS_SWIOTLB_DISABLED; #else - /* Old/new kernels, no way to detect, on x86 default to no IOMMU */ - state->state = 0; + /* No way to detect on old kernel */ + state->state = MODS_SWIOTLB_INDETERMINATE; #endif return OK; } +int esc_mods_pci_set_dma_mask(struct file *file, + struct MODS_PCI_DMA_MASK *dma_mask) +{ + int err; + unsigned int devfn = PCI_DEVFN(dma_mask->pci_device.device, + dma_mask->pci_device.function); + struct pci_dev *dev = MODS_PCI_GET_SLOT(dma_mask->pci_device.domain, + dma_mask->pci_device.bus, + devfn); + u64 mask; + + if (dma_mask->num_bits > 64) + return -EINVAL; + mask = dma_mask->num_bits == 64 ? ~0ULL : (1ULL<num_bits)-1; + + err = pci_set_dma_mask(dev, mask); + if (err) { + mods_error_printk("failed to set dma mask 0x%llx for dev %04x:%x:%02x.%x\n", + mask, + (unsigned int)dma_mask->pci_device.domain, + (unsigned int)dma_mask->pci_device.bus, + (unsigned int)dma_mask->pci_device.device, + (unsigned int)dma_mask->pci_device.function); +#if defined(CONFIG_PPC64) + /* Ignore error if TCE bypass is on */ + if (dev->dma_mask == ~0ULL) + err = OK; +#endif + } else + err = pci_set_consistent_dma_mask(dev, mask); + + return err; +}