diff --git a/drivers/pci/endpoint/functions/pci-epf-dma-test.c b/drivers/pci/endpoint/functions/pci-epf-dma-test.c index ac5fdc17..22058782 100644 --- a/drivers/pci/endpoint/functions/pci-epf-dma-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-dma-test.c @@ -67,822 +67,6 @@ struct pcie_epf_dma { struct edmalib_common edma; }; -struct edma_desc { - dma_addr_t src; - dma_addr_t dst; - size_t sz; -}; - -static irqreturn_t pcie_dma_epf_wr0_msi(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *)epfnv->bar0_virt; - - epf_bar0->wr_data[0].crc = crc32_le(~0, epfnv->bar0_virt + SZ_128M + BAR0_DMA_BUF_OFFSET, - epf_bar0->wr_data[0].size); - - return IRQ_HANDLED; -} - -static irqreturn_t pcie_dma_epf_wr1_msi(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - int bit = 1; - - epfnv->wr_busy &= ~BIT(bit); - wake_up(&epfnv->wr_wq[bit]); - - return IRQ_HANDLED; -} - -static irqreturn_t pcie_dma_epf_wr2_msi(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - int bit = 2; - - epfnv->wr_busy &= ~BIT(bit); - wake_up(&epfnv->wr_wq[bit]); - - return IRQ_HANDLED; -} - -static irqreturn_t pcie_dma_epf_wr3_msi(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - int bit = 3; - - epfnv->wr_busy &= ~BIT(bit); - wake_up(&epfnv->wr_wq[bit]); - - return IRQ_HANDLED; -} - -static irqreturn_t pcie_dma_epf_rd0_msi(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - int bit = 0; - - epfnv->rd_busy &= ~BIT(bit); - wake_up(&epfnv->rd_wq[bit]); - - return IRQ_HANDLED; -} - -static irqreturn_t pcie_dma_epf_rd1_msi(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - int bit = 1; - - epfnv->rd_busy &= ~BIT(bit); - wake_up(&epfnv->rd_wq[bit]); - - return IRQ_HANDLED; -} - -void pcie_async_dma_handler(struct pcie_epf_dma *epfnv) -{ - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - u64 llp_base, llp_iova; - u32 llp_idx, ridx; - int i; - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - llp_iova = dma_channel_rd(epfnv->dma_base, i, - DMA_LLP_HIGH_OFF_WRCH); - llp_iova = (llp_iova << 32); - llp_iova |= dma_channel_rd(epfnv->dma_base, i, - DMA_LLP_LOW_OFF_WRCH); - llp_base = epf_bar0->ep_phy_addr + DMA_LL_WR_OFFSET(i); - llp_idx = ((llp_iova - llp_base) / sizeof(struct dma_ll)); - llp_idx = llp_idx % DMA_ASYNC_LL_SIZE; - - if (!llp_idx) - continue; - - ridx = epfnv->rd_cnt[i] % DMA_ASYNC_LL_SIZE; - - while (llp_idx != ridx) { - epfnv->rd_cnt[i]++; - ridx = epfnv->rd_cnt[i] % DMA_ASYNC_LL_SIZE; - } - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - llp_iova = dma_channel_rd(epfnv->dma_base, i, - DMA_LLP_HIGH_OFF_RDCH); - llp_iova = (llp_iova << 32); - llp_iova |= dma_channel_rd(epfnv->dma_base, i, - DMA_LLP_LOW_OFF_RDCH); - llp_base = epf_bar0->ep_phy_addr + DMA_LL_RD_OFFSET(i); - llp_idx = ((llp_iova - llp_base) / sizeof(struct dma_ll)); - llp_idx = llp_idx % DMA_ASYNC_LL_SIZE; - - if (!llp_idx) - continue; - - ridx = epfnv->rd_cnt[DMA_WR_CHNL_NUM + i] % DMA_ASYNC_LL_SIZE; - - while (llp_idx != ridx) { - epfnv->rd_cnt[DMA_WR_CHNL_NUM + i]++; - ridx = epfnv->rd_cnt[DMA_WR_CHNL_NUM + i] % - DMA_ASYNC_LL_SIZE; - } - } -} - -static irqreturn_t pcie_dma_epf_irq(int irq, void *arg) -{ - return IRQ_WAKE_THREAD; -} - -static irqreturn_t pcie_dma_epf_irq_handler(int irq, void *arg) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg; - int bit = 0; - u32 val; - - val = dma_common_rd(epfnv->dma_base, DMA_WRITE_INT_STATUS_OFF); - for_each_set_bit(bit, &epfnv->wr_busy, DMA_WR_CHNL_NUM) { - if (BIT(bit) & val) { - dma_common_wr(epfnv->dma_base, BIT(bit), - DMA_WRITE_INT_CLEAR_OFF); - epfnv->wr_end_time[bit] = ktime_get(); - epfnv->wr_busy &= ~(BIT(bit)); - wake_up(&epfnv->wr_wq[bit]); - } - } - - val = dma_common_rd(epfnv->dma_base, DMA_READ_INT_STATUS_OFF); - for_each_set_bit(bit, &epfnv->rd_busy, DMA_RD_CHNL_NUM) { - if (BIT(bit) & val) { - dma_common_wr(epfnv->dma_base, BIT(bit), - DMA_READ_INT_CLEAR_OFF); - epfnv->rd_end_time[bit] = ktime_get(); - epfnv->rd_busy &= ~(BIT(bit)); - wake_up(&epfnv->rd_wq[bit]); - } - } - - if (epfnv->async_dma) { - val = dma_common_rd(epfnv->dma_base, DMA_WRITE_INT_STATUS_OFF); - dma_common_wr(epfnv->dma_base, val, DMA_WRITE_INT_CLEAR_OFF); - val = dma_common_rd(epfnv->dma_base, DMA_READ_INT_STATUS_OFF); - dma_common_wr(epfnv->dma_base, val, DMA_READ_INT_CLEAR_OFF); - pcie_async_dma_handler(epfnv); - } - - return IRQ_HANDLED; -} - -static int edma_init(struct pcie_epf_dma *epfnv, bool lie) -{ - u32 val; - int i; - - /* Enable LIE or RIE for all write channels */ - if (lie) { - val = dma_common_rd(epfnv->dma_base, DMA_WRITE_INT_MASK_OFF); - val &= ~0xf; - val &= ~(0xf << 16); - dma_common_wr(epfnv->dma_base, val, DMA_WRITE_INT_MASK_OFF); - } else { - val = dma_common_rd(epfnv->dma_base, DMA_WRITE_INT_MASK_OFF); - val |= 0xf; - val |= (0xf << 16); - dma_common_wr(epfnv->dma_base, val, DMA_WRITE_INT_MASK_OFF); - } - - val = DMA_CH_CONTROL1_OFF_WRCH_LIE; - if (!lie) - val |= DMA_CH_CONTROL1_OFF_WRCH_RIE; - for (i = 0; i < DMA_WR_CHNL_NUM; i++) - dma_channel_wr(epfnv->dma_base, i, val, - DMA_CH_CONTROL1_OFF_WRCH); - - /* Enable LIE or RIE for all read channels */ - if (lie) { - val = dma_common_rd(epfnv->dma_base, DMA_READ_INT_MASK_OFF); - val &= ~0x3; - val &= ~(0x3 << 16); - dma_common_wr(epfnv->dma_base, val, DMA_READ_INT_MASK_OFF); - } else { - val = dma_common_rd(epfnv->dma_base, DMA_READ_INT_MASK_OFF); - val |= 0x3; - val |= (0x3 << 16); - dma_common_wr(epfnv->dma_base, val, DMA_READ_INT_MASK_OFF); - } - - val = DMA_CH_CONTROL1_OFF_RDCH_LIE; - if (!lie) - val |= DMA_CH_CONTROL1_OFF_RDCH_RIE; - for (i = 0; i < DMA_RD_CHNL_NUM; i++) - dma_channel_wr(epfnv->dma_base, i, val, - DMA_CH_CONTROL1_OFF_RDCH); - - dma_common_wr(epfnv->dma_base, WRITE_ENABLE, DMA_WRITE_ENGINE_EN_OFF); - dma_common_wr(epfnv->dma_base, READ_ENABLE, DMA_READ_ENGINE_EN_OFF); - - return 0; -} - -static void edma_deinit(struct pcie_epf_dma *epfnv) -{ - u32 val; - - /* Mask channel interrupts */ - val = dma_common_rd(epfnv->dma_base, DMA_WRITE_INT_MASK_OFF); - val |= 0xf; - val |= (0xf << 16); - dma_common_wr(epfnv->dma_base, val, DMA_WRITE_INT_MASK_OFF); - - val = dma_common_rd(epfnv->dma_base, DMA_READ_INT_MASK_OFF); - val |= 0x3; - val |= (0x3 << 16); - dma_common_wr(epfnv->dma_base, val, DMA_READ_INT_MASK_OFF); - - dma_common_wr(epfnv->dma_base, WRITE_DISABLE, DMA_WRITE_ENGINE_EN_OFF); - dma_common_wr(epfnv->dma_base, READ_DISABLE, DMA_READ_ENGINE_EN_OFF); -} - -static int edma_ll_init(struct pcie_epf_dma *epfnv) -{ - u32 val; - int i; - - /* Enable linked list mode and set CCS */ - val = DMA_CH_CONTROL1_OFF_WRCH_LLE | DMA_CH_CONTROL1_OFF_WRCH_CCS; - for (i = 0; i < DMA_WR_CHNL_NUM; i++) - dma_channel_wr(epfnv->dma_base, i, val, - DMA_CH_CONTROL1_OFF_WRCH); - - val = DMA_CH_CONTROL1_OFF_RDCH_LLE | DMA_CH_CONTROL1_OFF_WRCH_CCS; - for (i = 0; i < DMA_RD_CHNL_NUM; i++) - dma_channel_wr(epfnv->dma_base, i, val, - DMA_CH_CONTROL1_OFF_RDCH); - - return 0; -} - -static void edma_ll_deinit(struct pcie_epf_dma *epfnv) -{ - u32 val; - int i; - - /* Disable linked list mode and clear CCS */ - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - val = dma_channel_rd(epfnv->dma_base, i, - DMA_CH_CONTROL1_OFF_WRCH); - val &= ~(DMA_CH_CONTROL1_OFF_WRCH_LLE | - DMA_CH_CONTROL1_OFF_WRCH_CCS); - dma_channel_wr(epfnv->dma_base, i, val, - DMA_CH_CONTROL1_OFF_WRCH); - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - val = dma_channel_rd(epfnv->dma_base, i, - DMA_CH_CONTROL1_OFF_RDCH); - val &= ~(DMA_CH_CONTROL1_OFF_RDCH_LLE | - DMA_CH_CONTROL1_OFF_RDCH_CCS); - dma_channel_wr(epfnv->dma_base, i, val, - DMA_CH_CONTROL1_OFF_RDCH); - } -} - -static int edma_submit_direct_tx(struct pcie_epf_dma *epfnv, - struct edma_desc *desc, int ch) -{ - int ret = 0; - - epfnv->wr_busy |= 1 << ch; - - /* Populate desc in DMA registers */ - dma_channel_wr(epfnv->dma_base, ch, desc->sz, - DMA_TRANSFER_SIZE_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, ch, lower_32_bits(desc->src), - DMA_SAR_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, ch, upper_32_bits(desc->src), - DMA_SAR_HIGH_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, ch, lower_32_bits(desc->dst), - DMA_DAR_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, ch, upper_32_bits(desc->dst), - DMA_DAR_HIGH_OFF_WRCH); - - epfnv->wr_start_time[ch] = ktime_get(); - dma_common_wr(epfnv->dma_base, ch, DMA_WRITE_DOORBELL_OFF); - - /* Wait 5 sec to get DMA done interrupt */ - ret = wait_event_timeout(epfnv->wr_wq[ch], - !(epfnv->wr_busy & (1 << ch)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: DD WR CH: %d TO\n", __func__, ch); - ret = -ETIMEDOUT; - } - - return ret; -} - -static int edma_submit_direct_rx(struct pcie_epf_dma *epfnv, - struct edma_desc *desc, int ch) -{ - int ret = 0; - - epfnv->rd_busy |= 1 << ch; - - /* Populate desc in DMA registers */ - dma_channel_wr(epfnv->dma_base, ch, desc->sz, - DMA_TRANSFER_SIZE_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, ch, lower_32_bits(desc->src), - DMA_SAR_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, ch, upper_32_bits(desc->src), - DMA_SAR_HIGH_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, ch, lower_32_bits(desc->dst), - DMA_DAR_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, ch, upper_32_bits(desc->dst), - DMA_DAR_HIGH_OFF_RDCH); - - epfnv->rd_start_time[ch] = ktime_get(); - dma_common_wr(epfnv->dma_base, ch, DMA_READ_DOORBELL_OFF); - - ret = wait_event_timeout(epfnv->rd_wq[ch], - !(epfnv->rd_busy & (1 << ch)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: DD RD CH: %d TO\n", - __func__, ch); - ret = -ETIMEDOUT; - } - - return ret; -} - -static int edma_submit_direct_txrx(struct pcie_epf_dma *epfnv, - struct edma_desc *desc_wr, - struct edma_desc *desc_rd) -{ - int ret = 0, i; - - /* Configure all DMA write and read channels */ - epfnv->wr_busy = DMA_WR_CHNL_MASK; - epfnv->rd_busy = DMA_RD_CHNL_MASK; - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - dma_channel_wr(epfnv->dma_base, i, desc_wr[i].sz, - DMA_TRANSFER_SIZE_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, i, - lower_32_bits(desc_wr[i].src), - DMA_SAR_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, i, - upper_32_bits(desc_wr[i].src), - DMA_SAR_HIGH_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, i, - lower_32_bits(desc_wr[i].dst), - DMA_DAR_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, i, - upper_32_bits(desc_wr[i].dst), - DMA_DAR_HIGH_OFF_WRCH); - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - dma_channel_wr(epfnv->dma_base, i, desc_rd[i].sz, - DMA_TRANSFER_SIZE_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, i, - lower_32_bits(desc_rd[i].src), - DMA_SAR_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, i, - upper_32_bits(desc_rd[i].src), - DMA_SAR_HIGH_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, i, - lower_32_bits(desc_rd[i].dst), - DMA_DAR_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, i, - upper_32_bits(desc_rd[i].dst), - DMA_DAR_HIGH_OFF_RDCH); - } - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - dma_common_wr(epfnv->dma_base, i, DMA_WRITE_DOORBELL_OFF); - if (i < DMA_RD_CHNL_NUM) - dma_common_wr(epfnv->dma_base, i, - DMA_READ_DOORBELL_OFF); - } - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - ret = wait_event_timeout(epfnv->wr_wq[i], - !(epfnv->wr_busy & (1 << i)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: DD WR CH: %d TO\n", - __func__, i); - ret = -ETIMEDOUT; - goto fail; - } - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - ret = wait_event_timeout(epfnv->rd_wq[i], - !(epfnv->rd_busy & (1 << i)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: DD RD CH: %d TO\n", - __func__, i); - ret = -ETIMEDOUT; - goto fail; - } - } - -fail: - return ret; -} - -static int edma_submit_sync_tx(struct pcie_epf_dma *epfnv, - struct edma_desc *desc, - int nents, int ch, bool lie) -{ - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - dma_addr_t ll_phy_addr = epf_bar0->ep_phy_addr + DMA_LL_WR_OFFSET(ch); - struct dma_ll *dma_ll_virt; - int i, ret; - - epfnv->wr_busy |= 1 << ch; - - /* Program DMA LL base address in DMA LL pointer register */ - dma_channel_wr(epfnv->dma_base, ch, lower_32_bits(ll_phy_addr), - DMA_LLP_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, ch, upper_32_bits(ll_phy_addr), - DMA_LLP_HIGH_OFF_WRCH); - - /* Populate DMA descriptors in LL */ - dma_ll_virt = (struct dma_ll *) - (epfnv->bar0_virt + DMA_LL_WR_OFFSET(ch)); - for (i = 0; i < nents; i++) { - dma_ll_virt->size = desc[i].sz; - dma_ll_virt->src_low = lower_32_bits(desc[i].src); - dma_ll_virt->src_high = upper_32_bits(desc[i].src); - dma_ll_virt->dst_low = lower_32_bits(desc[i].dst); - dma_ll_virt->dst_high = upper_32_bits(desc[i].dst); - dma_ll_virt->ele.cb = 1; - dma_ll_virt++; - } - /* Set LIE or RIE in last element */ - dma_ll_virt--; - dma_ll_virt->ele.lie = 1; - if (!lie) - dma_ll_virt->ele.rie = 1; - - epfnv->wr_start_time[ch] = ktime_get(); - dma_common_wr(epfnv->dma_base, ch, DMA_WRITE_DOORBELL_OFF); - - ret = wait_event_timeout(epfnv->wr_wq[ch], - !(epfnv->wr_busy & (1 << ch)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: LL WR CH: %d TO\n", __func__, ch); - ret = -ETIMEDOUT; - } - - return ret; -} - -static int edma_submit_sync_rx(struct pcie_epf_dma *epfnv, - struct edma_desc *desc, - int nents, int ch, bool lie) -{ - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - dma_addr_t ll_phy_addr = epf_bar0->ep_phy_addr + DMA_LL_RD_OFFSET(ch); - struct dma_ll *dma_ll_virt; - int i, ret; - - epfnv->rd_busy |= 1 << ch; - - /* Program DMA LL base address in DMA LL pointer register */ - dma_channel_wr(epfnv->dma_base, ch, lower_32_bits(ll_phy_addr), - DMA_LLP_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, ch, upper_32_bits(ll_phy_addr), - DMA_LLP_HIGH_OFF_RDCH); - - /* Populate DMA descriptors in LL */ - dma_ll_virt = (struct dma_ll *) - (epfnv->bar0_virt + DMA_LL_RD_OFFSET(ch)); - for (i = 0; i < nents; i++) { - dma_ll_virt->size = desc[i].sz; - dma_ll_virt->src_low = lower_32_bits(desc[i].src); - dma_ll_virt->src_high = upper_32_bits(desc[i].src); - dma_ll_virt->dst_low = lower_32_bits(desc[i].dst); - dma_ll_virt->dst_high = upper_32_bits(desc[i].dst); - dma_ll_virt->ele.cb = 1; - dma_ll_virt++; - } - /* Set LIE or RIE in last element */ - dma_ll_virt--; - dma_ll_virt->ele.lie = 1; - if (!lie) - dma_ll_virt->ele.rie = 1; - - epfnv->rd_start_time[ch] = ktime_get(); - dma_common_wr(epfnv->dma_base, ch, DMA_READ_DOORBELL_OFF); - - ret = wait_event_timeout(epfnv->rd_wq[ch], - !(epfnv->rd_busy & (1 << ch)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: LL RD CH: %d TO\n", __func__, ch); - ret = -ETIMEDOUT; - } - - return ret; -} - -static int edma_submit_sync_txrx(struct pcie_epf_dma *epfnv, - struct edma_desc *desc_wr, - struct edma_desc *desc_rd, int nents) -{ - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - dma_addr_t phy_addr = epf_bar0->ep_phy_addr; - struct dma_ll *dma_ll_virt; - int i, j, ret; - - epfnv->wr_busy = DMA_WR_CHNL_MASK; - epfnv->rd_busy = DMA_RD_CHNL_MASK; - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - dma_channel_wr(epfnv->dma_base, i, - lower_32_bits(phy_addr + DMA_LL_WR_OFFSET(i)), - DMA_LLP_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, i, - upper_32_bits(phy_addr + DMA_LL_WR_OFFSET(i)), - DMA_LLP_HIGH_OFF_WRCH); - - dma_ll_virt = (struct dma_ll *) - (epfnv->bar0_virt + DMA_LL_WR_OFFSET(i)); - for (j = i * nents; j < ((i + 1) * nents); j++) { - dma_ll_virt->size = desc_wr[j].sz; - dma_ll_virt->src_low = lower_32_bits(desc_wr[j].src); - dma_ll_virt->src_high = upper_32_bits(desc_wr[j].src); - dma_ll_virt->dst_low = lower_32_bits(desc_wr[j].dst); - dma_ll_virt->dst_high = upper_32_bits(desc_wr[j].dst); - dma_ll_virt->ele.cb = 1; - dma_ll_virt++; - } - dma_ll_virt--; - dma_ll_virt->ele.lie = 1; - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - dma_channel_wr(epfnv->dma_base, i, - lower_32_bits(phy_addr + DMA_LL_RD_OFFSET(i)), - DMA_LLP_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, i, - upper_32_bits(phy_addr + DMA_LL_RD_OFFSET(i)), - DMA_LLP_HIGH_OFF_RDCH); - - dma_ll_virt = (struct dma_ll *) - (epfnv->bar0_virt + DMA_LL_RD_OFFSET(i)); - for (j = i * nents; j < ((i + 1) * nents); j++) { - dma_ll_virt->size = desc_rd[j].sz; - dma_ll_virt->src_low = lower_32_bits(desc_rd[j].src); - dma_ll_virt->src_high = upper_32_bits(desc_rd[j].src); - dma_ll_virt->dst_low = lower_32_bits(desc_rd[j].dst); - dma_ll_virt->dst_high = upper_32_bits(desc_rd[j].dst); - dma_ll_virt->ele.cb = 1; - dma_ll_virt++; - } - dma_ll_virt--; - dma_ll_virt->ele.lie = 1; - } - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - dma_common_wr(epfnv->dma_base, i, DMA_WRITE_DOORBELL_OFF); - if (i < DMA_RD_CHNL_NUM) - dma_common_wr(epfnv->dma_base, i, - DMA_READ_DOORBELL_OFF); - } - - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - ret = wait_event_timeout(epfnv->wr_wq[i], - !(epfnv->wr_busy & (1 << i)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: LL WR CH: %d TO\n", - __func__, i); - ret = -ETIMEDOUT; - goto fail; - } - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - ret = wait_event_timeout(epfnv->rd_wq[i], - !(epfnv->rd_busy & (1 << i)), - msecs_to_jiffies(5000)); - if (ret == 0) { - dev_err(epfnv->fdev, "%s: LL RD CH: %d TO\n", - __func__, i); - ret = -ETIMEDOUT; - goto fail; - } - } - -fail: - return ret; -} - -/* debugfs to measure direct and LL DMA read/write perf on channel 0 */ -static int perf_test(struct seq_file *s, void *data) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *) - dev_get_drvdata(s->private); - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - struct edma_desc desc; - struct edma_desc ll_desc[DMA_LL_DEFAULT_SIZE]; - dma_addr_t ep_dma_addr = epf_bar0->ep_phy_addr + BAR0_DMA_BUF_OFFSET; - dma_addr_t rp_dma_addr = epf_bar0->rp_phy_addr + BAR0_DMA_BUF_OFFSET; - long long time; - int ch = 0, nents = DMA_LL_MIN_SIZE, i, ret; - - if (!rp_dma_addr) { - dev_err(epfnv->fdev, "RP DMA address is null\n"); - return 0; - } - - edma_init(epfnv, 1); - - /* Direct DMA perf test with size BAR0_DMA_BUF_SIZE */ - desc.src = ep_dma_addr; - desc.dst = rp_dma_addr; - desc.sz = BAR0_DMA_BUF_SIZE; - ret = edma_submit_direct_tx(epfnv, &desc, ch); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: DD WR, SZ: %lu B CH: %d failed\n", - __func__, desc.sz, ch); - goto fail; - } - - time = ktime_to_ns(epfnv->wr_end_time[ch]) - - ktime_to_ns(epfnv->wr_start_time[ch]); - dev_info(epfnv->fdev, "%s: DD WR, CH: %d SZ: %lu B, time: %lld ns\n", - __func__, ch, desc.sz, time); - - desc.src = rp_dma_addr; - desc.dst = ep_dma_addr; - desc.sz = BAR0_DMA_BUF_SIZE; - ret = edma_submit_direct_rx(epfnv, &desc, ch); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: DD RD, SZ: %lu B CH: %d failed\n", - __func__, desc.sz, ch); - goto fail; - } - time = ktime_to_ns(epfnv->rd_end_time[ch]) - - ktime_to_ns(epfnv->rd_start_time[ch]); - dev_info(epfnv->fdev, "%s: DD RD, CH: %d SZ: %lu B, time: %lld ns\n", - __func__, ch, desc.sz, time); - - /* Clean DMA LL */ - memset(epfnv->bar0_virt + DMA_LL_WR_OFFSET(0), 0, 6 * DMA_LL_SIZE); - edma_ll_init(epfnv); - - /* LL DMA perf test with size BAR0_DMA_BUF_SIZE and one desc */ - for (i = 0; i < nents; i++) { - ll_desc[i].src = ep_dma_addr + (i * BAR0_DMA_BUF_SIZE); - ll_desc[i].dst = rp_dma_addr + (i * BAR0_DMA_BUF_SIZE); - ll_desc[i].sz = BAR0_DMA_BUF_SIZE; - } - - ret = edma_submit_sync_tx(epfnv, ll_desc, nents, ch, 1); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: LL WR, SZ: %u B CH: %d failed\n", - __func__, BAR0_DMA_BUF_SIZE * nents, ch); - goto fail; - } - time = ktime_to_ns(epfnv->wr_end_time[ch]) - - ktime_to_ns(epfnv->wr_start_time[ch]); - dev_info(epfnv->fdev, "%s: LL WR, CH: %d N: %d SZ: %d B, time: %lld ns\n", - __func__, ch, nents, BAR0_DMA_BUF_SIZE, time); - - for (i = 0; i < nents; i++) { - ll_desc[i].src = rp_dma_addr + (i * BAR0_DMA_BUF_SIZE); - ll_desc[i].dst = ep_dma_addr + (i * BAR0_DMA_BUF_SIZE); - ll_desc[i].sz = BAR0_DMA_BUF_SIZE; - } - - ret = edma_submit_sync_rx(epfnv, ll_desc, nents, ch, 1); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: LL RD, SZ: %u B CH: %d failed\n", - __func__, BAR0_DMA_BUF_SIZE * nents, ch); - goto fail; - } - time = ktime_to_ns(epfnv->rd_end_time[ch]) - - ktime_to_ns(epfnv->rd_start_time[ch]); - dev_info(epfnv->fdev, "%s: LL RD, CH: %d N: %d SZ: %d B, time: %lld ns\n", - __func__, ch, nents, BAR0_DMA_BUF_SIZE, time); - - edma_ll_deinit(epfnv); - edma_deinit(epfnv); - -fail: - return 0; -} - -/* debugfs to stress direct and LL DMA on all wr & rd channels */ -static int stress_test(struct seq_file *s, void *data) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *) - dev_get_drvdata(s->private); - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - struct edma_desc desc_wr[DMA_WR_CHNL_NUM], desc_rd[DMA_RD_CHNL_NUM]; - struct edma_desc ll_desc_wr[DMA_WR_CHNL_NUM * DMA_LL_DEFAULT_SIZE]; - struct edma_desc ll_desc_rd[DMA_RD_CHNL_NUM * DMA_LL_DEFAULT_SIZE]; - dma_addr_t ep_dma_addr = epf_bar0->ep_phy_addr + BAR0_DMA_BUF_OFFSET; - dma_addr_t rp_dma_addr = epf_bar0->rp_phy_addr + BAR0_DMA_BUF_OFFSET; - int i, j, ret, nents = DMA_LL_DEFAULT_SIZE; - - if (!rp_dma_addr) { - dev_err(epfnv->fdev, "RP DMA address is null\n"); - return 0; - } - - edma_init(epfnv, 1); - - /* Direct DMA stress test with rand size < DMA_DD_BUF_SIZE */ - for (j = 0; j < DMA_WR_CHNL_NUM; j++) { - desc_wr[j].src = ep_dma_addr + (j * DMA_DD_BUF_SIZE); - desc_wr[j].dst = rp_dma_addr + (j * DMA_DD_BUF_SIZE); - } - - for (j = 0; j < DMA_RD_CHNL_NUM; j++) { - desc_rd[j].src = rp_dma_addr + - ((j + DMA_WR_CHNL_NUM) * DMA_DD_BUF_SIZE); - desc_rd[j].dst = ep_dma_addr + - ((j + DMA_WR_CHNL_NUM) * DMA_DD_BUF_SIZE); - } - - for (i = 0; i < epfnv->stress_count; i++) { - for (j = 0; j < DMA_WR_CHNL_NUM; j++) - desc_wr[j].sz = - (get_random_u32() % DMA_DD_BUF_SIZE) + 1; - for (j = 0; j < DMA_RD_CHNL_NUM; j++) - desc_rd[j].sz = - (get_random_u32() % DMA_DD_BUF_SIZE) + 1; - ret = edma_submit_direct_txrx(epfnv, desc_wr, desc_rd); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: DD stress failed\n", - __func__); - goto fail; - } - dev_info(epfnv->fdev, "%s: DD stress test iteration %d done\n", - __func__, i); - } - dev_info(epfnv->fdev, "%s: DD stress: all CH, rand SZ, count: %d success\n", - __func__, epfnv->stress_count); - - /* Clean DMA LL */ - memset(epfnv->bar0_virt + DMA_LL_WR_OFFSET(0), 0, 6 * DMA_LL_SIZE); - edma_ll_init(epfnv); - - /* LL DMA stress test with rand size < DMA_LL_BUF_SIZE per desc */ - for (i = 0; i < DMA_WR_CHNL_NUM * nents; i++) { - ll_desc_wr[i].src = ep_dma_addr + (i * DMA_LL_BUF_SIZE); - ll_desc_wr[i].dst = rp_dma_addr + (i * DMA_LL_BUF_SIZE); - } - for (i = 0; i < DMA_RD_CHNL_NUM * nents; i++) { - ll_desc_rd[i].src = rp_dma_addr + - ((i + DMA_WR_CHNL_NUM) * DMA_LL_BUF_SIZE); - ll_desc_rd[i].dst = ep_dma_addr + - ((i + DMA_WR_CHNL_NUM) * DMA_LL_BUF_SIZE); - } - - for (i = 0; i < epfnv->stress_count; i++) { - for (j = 0; j < DMA_WR_CHNL_NUM * nents; j++) - ll_desc_wr[j].sz = - (get_random_u32() % DMA_LL_BUF_SIZE) + 1; - for (j = 0; j < DMA_RD_CHNL_NUM * nents; j++) - ll_desc_rd[j].sz = - (get_random_u32() % DMA_LL_BUF_SIZE) + 1; - ret = edma_submit_sync_txrx(epfnv, ll_desc_wr, ll_desc_rd, - nents); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: DMA LL stress failed\n", - __func__); - goto fail; - } - dev_info(epfnv->fdev, "%s: LL stress test iteration %d done\n", - __func__, i); - } - dev_info(epfnv->fdev, "%s: LL stress: all CH, rand SZ, count: %d success\n", - __func__, epfnv->stress_count); - - edma_ll_deinit(epfnv); - edma_deinit(epfnv); - -fail: - return 0; -} - static void edma_lib_test_raise_irq(void *p) { struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)p; @@ -920,454 +104,8 @@ static int edmalib_test(struct seq_file *s, void *data) return edmalib_common_test(&epfnv->edma); } -/* debugfs to perform direct & LL DMA and do CRC check */ -static int sanity_test(struct seq_file *s, void *data) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *) - dev_get_drvdata(s->private); - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - struct edma_desc desc; - struct edma_desc ll_desc[DMA_LL_DEFAULT_SIZE]; - dma_addr_t ep_dma_addr = epf_bar0->ep_phy_addr + BAR0_DMA_BUF_OFFSET; - dma_addr_t rp_dma_addr = epf_bar0->rp_phy_addr + BAR0_DMA_BUF_OFFSET; - int nents = DMA_LL_DEFAULT_SIZE; - int i, j, ret; - u32 crc; - - if (epfnv->dma_size > MAX_DMA_ELE_SIZE) { - dev_err(epfnv->fdev, "%s: dma_size should be <= 0x%x\n", - __func__, MAX_DMA_ELE_SIZE); - goto fail; - } - - if (!rp_dma_addr) { - dev_err(epfnv->fdev, "RP DMA address is null\n"); - return 0; - } - - edma_init(epfnv, 0); - - /* Direct DMA of size epfnv->dma_size */ - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - desc.src = ep_dma_addr; - desc.dst = rp_dma_addr; - desc.sz = epfnv->dma_size; - epf_bar0->wr_data[i].size = desc.sz; - /* generate random bytes to transfer */ - get_random_bytes(epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, - desc.sz); - ret = edma_submit_direct_tx(epfnv, &desc, i); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: DD WR CH: %d failed\n", - __func__, i); - goto fail; - } - crc = crc32_le(~0, epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, - desc.sz); - if (crc != epf_bar0->wr_data[i].crc) { - dev_err(epfnv->fdev, "%s: DD WR, SZ: %lu B CH: %d CRC failed\n", - __func__, desc.sz, i); - goto fail; - } - dev_info(epfnv->fdev, "%s: DD WR, SZ: %lu B CH: %d success\n", - __func__, desc.sz, i); - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - desc.src = rp_dma_addr; - desc.dst = ep_dma_addr; - desc.sz = epfnv->dma_size; - epf_bar0->rd_data[i].size = desc.sz; - /* Clear memory to receive data */ - memset(epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, 0, desc.sz); - ret = edma_submit_direct_rx(epfnv, &desc, i); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: DD RD CH: %d failed\n", - __func__, i); - goto fail; - } - crc = crc32_le(~0, epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, - desc.sz); - if (crc != epf_bar0->rd_data[i].crc) { - dev_err(epfnv->fdev, "%s: DD RD, SZ: %lu B CH: %d CRC failed\n", - __func__, desc.sz, i); - goto fail; - } - dev_info(epfnv->fdev, "%s: DD RD, SZ: %lu B CH: %d success\n", - __func__, desc.sz, i); - } - - /* Clean DMA LL all 6 channels */ - memset(epfnv->bar0_virt + DMA_LL_WR_OFFSET(0), 0, 6 * DMA_LL_SIZE); - edma_ll_init(epfnv); - - /* LL DMA with size epfnv->dma_size per desc */ - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - for (j = 0; j < nents; j++) { - ll_desc[j].src = ep_dma_addr + (j * epfnv->dma_size); - ll_desc[j].dst = rp_dma_addr + (j * epfnv->dma_size); - ll_desc[j].sz = epfnv->dma_size; - } - epf_bar0->wr_data[i].size = epfnv->dma_size * nents; - /* generate random bytes to transfer */ - get_random_bytes(epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, - epf_bar0->wr_data[i].size); - - ret = edma_submit_sync_tx(epfnv, ll_desc, nents, i, 0); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: LL WR CH: %d failed\n", - __func__, i); - goto fail; - } - crc = crc32_le(~0, epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, - epfnv->dma_size * nents); - if (crc != epf_bar0->wr_data[i].crc) { - dev_err(epfnv->fdev, "%s: LL WR, SZ: %u B CH: %d CRC failed\n", - __func__, epfnv->dma_size, i); - goto fail; - } - dev_info(epfnv->fdev, "%s: LL WR, SZ: %u B CH: %d success\n", - __func__, epfnv->dma_size, i); - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - for (j = 0; j < nents; j++) { - ll_desc[j].src = rp_dma_addr + (j * epfnv->dma_size); - ll_desc[j].dst = ep_dma_addr + (j * epfnv->dma_size); - ll_desc[j].sz = epfnv->dma_size; - } - epf_bar0->rd_data[i].size = epfnv->dma_size * nents; - /* Clear memory to receive data */ - memset(epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, 0, - epf_bar0->rd_data[i].size); - - ret = edma_submit_sync_rx(epfnv, ll_desc, nents, i, 0); - if (ret < 0) { - dev_err(epfnv->fdev, "%s: LL RD failed\n", __func__); - goto fail; - } - crc = crc32_le(~0, epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET, - epfnv->dma_size * nents); - if (crc != epf_bar0->rd_data[i].crc) { - dev_err(epfnv->fdev, "%s: LL RD, SZ: %u B CH: %d CRC failed\n", - __func__, epfnv->dma_size, i); - goto fail; - } - dev_info(epfnv->fdev, "%s: LL RD, SZ: %u B CH: %d success\n", - __func__, epfnv->dma_size, i); - } - - edma_ll_deinit(epfnv); - edma_deinit(epfnv); - -fail: - return 0; -} - -#ifdef THREAD_ON_CPU -static int async_dma_test_fn(struct pcie_epf_dma *epfnv, int ch) -{ - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - dma_addr_t ep_dma_addr, rp_dma_addr, phy_addr; - struct dma_ll *dma_ll_virt; - u32 nents = epfnv->async_count, count = 0, idx, i; - - ep_dma_addr = epf_bar0->ep_phy_addr + BAR0_DMA_BUF_OFFSET + - (ch * DMA_ASYNC_LL_SIZE * SZ_64K); - rp_dma_addr = epf_bar0->rp_phy_addr + BAR0_DMA_BUF_OFFSET + - (ch * DMA_ASYNC_LL_SIZE * SZ_64K); - - epfnv->wr_cnt[ch] = 0; - epfnv->rd_cnt[ch] = 0; - - dma_ll_virt = (struct dma_ll *) - (epfnv->bar0_virt + DMA_LL_WR_OFFSET(ch)); - phy_addr = epf_bar0->ep_phy_addr + DMA_LL_WR_OFFSET(ch); - dma_ll_virt[DMA_ASYNC_LL_SIZE].src_low = lower_32_bits(phy_addr); - dma_ll_virt[DMA_ASYNC_LL_SIZE].src_high = upper_32_bits(phy_addr); - dma_ll_virt[DMA_ASYNC_LL_SIZE].ele.llp = 1; - dma_ll_virt[DMA_ASYNC_LL_SIZE].ele.tcb = 1; - epfnv->pcs[ch] = 1; - dma_ll_virt[DMA_ASYNC_LL_SIZE].ele.cb = !epfnv->pcs[ch]; - - for (i = 0; i < nents; i++) { - while ((epfnv->wr_cnt[ch] - epfnv->rd_cnt[ch] + 2) >= - DMA_ASYNC_LL_SIZE) { - msleep(100); - if (++count == 100) { - dev_info(epfnv->fdev, "%s: CH: %d LL is full wr_cnt: %u rd_cnt: %u\n", - __func__, ch, epfnv->wr_cnt[ch], - epfnv->rd_cnt[ch]); - goto fail; - } - } - count = 0; - - idx = i % DMA_ASYNC_LL_SIZE; - - dma_ll_virt[idx].size = SZ_64K; - if (ch < DMA_WR_CHNL_NUM) { - phy_addr = ep_dma_addr + - (idx % DMA_ASYNC_LL_SIZE) * SZ_64K; - dma_ll_virt[idx].src_low = lower_32_bits(phy_addr); - dma_ll_virt[idx].src_high = upper_32_bits(phy_addr); - phy_addr = rp_dma_addr + - (idx % DMA_ASYNC_LL_SIZE) * SZ_64K; - dma_ll_virt[idx].dst_low = lower_32_bits(phy_addr); - dma_ll_virt[idx].dst_high = upper_32_bits(phy_addr); - } else { - phy_addr = rp_dma_addr + - (idx % DMA_ASYNC_LL_SIZE) * SZ_64K; - dma_ll_virt[idx].src_low = lower_32_bits(phy_addr); - dma_ll_virt[idx].src_high = upper_32_bits(phy_addr); - phy_addr = ep_dma_addr + - (idx % DMA_ASYNC_LL_SIZE) * SZ_64K; - dma_ll_virt[idx].dst_low = lower_32_bits(phy_addr); - dma_ll_virt[idx].dst_high = upper_32_bits(phy_addr); - } - dma_ll_virt[idx].ele.lie = 1; - /* - * DMA desc should not be touched after CB bit set, add - * write barrier to stop desc writes going out of order - * wrt CB bit set. - */ - wmb(); - dma_ll_virt[idx].ele.cb = epfnv->pcs[ch]; - if (idx == (DMA_ASYNC_LL_SIZE - 1)) { - epfnv->pcs[ch] = !epfnv->pcs[ch]; - dma_ll_virt[idx + 1].ele.cb = epfnv->pcs[ch]; - } - if (ch < DMA_WR_CHNL_NUM) - dma_common_wr(epfnv->dma_base, ch, - DMA_WRITE_DOORBELL_OFF); - else - dma_common_wr(epfnv->dma_base, ch - DMA_WR_CHNL_NUM, - DMA_READ_DOORBELL_OFF); - epfnv->wr_cnt[ch]++; - /* Print status for very 10000 iterations */ - if ((i % 10000) == 0) - dev_info(epfnv->fdev, "%s: CH: %u async DMA test itr: %u done, wr_cnt: %u rd_cnt: %u\n", - __func__, ch, i, epfnv->wr_cnt[ch], - epfnv->rd_cnt[ch]); - } - count = 0; - while (epfnv->wr_cnt[ch] != epfnv->rd_cnt[ch]) { - msleep(20); - if (++count == 100) { - dev_info(epfnv->fdev, "%s: CH: %d async DMA test failed, wr_cnt: %u rd_cnt: %u\n", - __func__, ch, epfnv->wr_cnt[ch], - epfnv->rd_cnt[ch]); - goto fail; - } - } - dev_info(epfnv->fdev, "%s: CH: %d async DMA success\n", __func__, ch); - -fail: - epfnv->wr_cnt[ch] = 0; - epfnv->rd_cnt[ch] = 0; - - return 0; -} - -static int async_wr0_work(void *data) -{ - struct pcie_epf_dma *epfnv = data; - - async_dma_test_fn(epfnv, 0); - - epfnv->task_done++; - wake_up(&epfnv->task_wq); - - return 0; -} - -static int async_wr1_work(void *data) -{ - struct pcie_epf_dma *epfnv = data; - - async_dma_test_fn(epfnv, 1); - - epfnv->task_done++; - wake_up(&epfnv->task_wq); - - return 0; -} - -static int async_wr2_work(void *data) -{ - struct pcie_epf_dma *epfnv = data; - - async_dma_test_fn(epfnv, 2); - - epfnv->task_done++; - wake_up(&epfnv->task_wq); - - return 0; -} - -static int async_wr3_work(void *data) -{ - struct pcie_epf_dma *epfnv = data; - - async_dma_test_fn(epfnv, 3); - - epfnv->task_done++; - wake_up(&epfnv->task_wq); - - return 0; -} - -static int async_rd0_work(void *data) -{ - struct pcie_epf_dma *epfnv = data; - - async_dma_test_fn(epfnv, 4); - - epfnv->task_done++; - wake_up(&epfnv->task_wq); - - return 0; -} - -static int async_rd1_work(void *data) -{ - struct pcie_epf_dma *epfnv = data; - - async_dma_test_fn(epfnv, 5); - - epfnv->task_done++; - wake_up(&epfnv->task_wq); - - return 0; -} -#endif -static int async_dma_test(struct seq_file *s, void *data) -{ - struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *) - dev_get_drvdata(s->private); - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - epfnv->bar0_virt; - dma_addr_t phy_addr; - int i; - - epfnv->task_done = 0; - epfnv->async_dma = true; - - edma_init(epfnv, 1); - /* Clean DMA LL all 6 channels */ - memset(epfnv->bar0_virt + DMA_LL_WR_OFFSET(0), 0, 6 * DMA_LL_SIZE); - edma_ll_init(epfnv); - - /* Program DMA LL base address in DMA LL pointer register */ - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - phy_addr = epf_bar0->ep_phy_addr + DMA_LL_WR_OFFSET(i); - dma_channel_wr(epfnv->dma_base, i, lower_32_bits(phy_addr), - DMA_LLP_LOW_OFF_WRCH); - dma_channel_wr(epfnv->dma_base, i, upper_32_bits(phy_addr), - DMA_LLP_HIGH_OFF_WRCH); - } - - for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - phy_addr = epf_bar0->ep_phy_addr + DMA_LL_RD_OFFSET(i); - dma_channel_wr(epfnv->dma_base, i, lower_32_bits(phy_addr), - DMA_LLP_LOW_OFF_RDCH); - dma_channel_wr(epfnv->dma_base, i, upper_32_bits(phy_addr), - DMA_LLP_HIGH_OFF_RDCH); - } - -#ifdef THREAD_ON_CPU - epfnv->wr0_task = kthread_create_on_cpu(async_wr0_work, epfnv, 0, - "async_wr0_work"); - if (IS_ERR(epfnv->wr0_task)) { - dev_err(epfnv->fdev, "failed to create async_wr0 thread\n"); - goto wr0_task; - } - epfnv->wr1_task = kthread_create_on_cpu(async_wr1_work, epfnv, 1, - "async_wr1_work"); - if (IS_ERR(epfnv->wr1_task)) { - dev_err(epfnv->fdev, "failed to create async_wr1 thread\n"); - goto wr1_task; - } - epfnv->wr2_task = kthread_create_on_cpu(async_wr2_work, epfnv, 2, - "async_wr2_work"); - if (IS_ERR(epfnv->wr2_task)) { - dev_err(epfnv->fdev, "failed to create async_wr2 thread\n"); - goto wr2_task; - } - epfnv->wr3_task = kthread_create_on_cpu(async_wr3_work, epfnv, 3, - "async_wr3_work"); - if (IS_ERR(epfnv->wr3_task)) { - dev_err(epfnv->fdev, "failed to create async_wr3 thread\n"); - goto wr3_task; - } - epfnv->rd0_task = kthread_create_on_cpu(async_rd0_work, epfnv, 4, - "async_rd0_work"); - if (IS_ERR(epfnv->rd0_task)) { - dev_err(epfnv->fdev, "failed to create async_rd0 thread\n"); - goto rd0_task; - } - epfnv->rd1_task = kthread_create_on_cpu(async_rd1_work, epfnv, 5, - "async_rd1_work"); - if (IS_ERR(epfnv->rd1_task)) { - dev_err(epfnv->fdev, "failed to create async_rd1 thread\n"); - goto rd1_task; - } -#endif - init_waitqueue_head(&epfnv->task_wq); - - wake_up_process(epfnv->wr0_task); - wake_up_process(epfnv->wr1_task); - wake_up_process(epfnv->wr2_task); - wake_up_process(epfnv->wr3_task); - wake_up_process(epfnv->rd0_task); - wake_up_process(epfnv->rd1_task); - - wait_event(epfnv->task_wq, - (epfnv->task_done == (DMA_WR_CHNL_NUM + DMA_RD_CHNL_NUM))); - dev_info(epfnv->fdev, "%s: Async DMA test done\n", __func__); - - edma_ll_deinit(epfnv); - edma_deinit(epfnv); - - epfnv->async_dma = false; - epfnv->task_done = 0; - - return 0; - -#ifdef THREAD_ON_CPU -rd1_task: - kthread_stop(epfnv->rd0_task); -rd0_task: - kthread_stop(epfnv->wr3_task); -wr3_task: - kthread_stop(epfnv->wr2_task); -wr2_task: - kthread_stop(epfnv->wr1_task); -wr1_task: - kthread_stop(epfnv->wr0_task); -wr0_task: - epfnv->async_dma = false; - epfnv->task_done = 0; - - return 0; -#endif -} - static void init_debugfs(struct pcie_epf_dma *epfnv) { - debugfs_create_devm_seqfile(epfnv->fdev, "perf_test", epfnv->debugfs, - perf_test); - - debugfs_create_devm_seqfile(epfnv->fdev, "stress_test", epfnv->debugfs, - stress_test); - - debugfs_create_devm_seqfile(epfnv->fdev, "sanity_test", epfnv->debugfs, - sanity_test); - - debugfs_create_devm_seqfile(epfnv->fdev, "async_dma_test", - epfnv->debugfs, async_dma_test); debugfs_create_devm_seqfile(epfnv->fdev, "edmalib_test", epfnv->debugfs, edmalib_test); @@ -1392,27 +130,6 @@ static void init_debugfs(struct pcie_epf_dma *epfnv) debugfs_create_u32("stress_count", 0644, epfnv->debugfs, &epfnv->stress_count); epfnv->stress_count = DEFAULT_STRESS_COUNT; - - debugfs_create_u32("async_count", 0644, epfnv->debugfs, - &epfnv->async_count); - epfnv->async_count = 4096; -} - -static void pcie_dma_epf_write_msi_msg(struct msi_desc *desc, - struct msi_msg *msg) -{ - /* TODO get rid of global variable gepfnv */ - struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) - gepfnv->bar0_virt; - struct device *cdev = msi_desc_to_dev(desc); -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0) - int idx = desc->platform.msi_index; -#else - int idx = desc->msi_index; -#endif - - epf_bar0->msi_data[idx] = msg->data; - dev_info(cdev, "%s: MSI idx: %d data: %d\n", __func__, idx, msg->data); } static int pcie_dma_epf_core_init(struct pci_epf *epf) @@ -1447,104 +164,6 @@ static int pcie_dma_epf_core_init(struct pci_epf *epf) return 0; } -static int pcie_dma_epf_msi_init(struct pci_epf *epf) -{ - struct pcie_epf_dma *epfnv = epf_get_drvdata(epf); - struct pci_epc *epc = epf->epc; - struct device *cdev = epc->dev.parent; - struct device *fdev = &epf->dev; - struct msi_desc *desc; - int ret; - - /* LL DMA in sanity test will not work without MSI for EP */ - if (!dev_get_msi_domain(cdev)) { - dev_info(fdev, "msi_domain absent, no interrupts\n"); - return 0; - } - ret = platform_msi_domain_alloc_irqs(cdev, - DMA_WR_CHNL_NUM + DMA_RD_CHNL_NUM, - pcie_dma_epf_write_msi_msg); - if (ret < 0) { - dev_err(fdev, "failed to allocate MSIs\n"); - return ret; - } - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0) - for_each_msi_entry(desc, cdev) { - switch (desc->platform.msi_index) { -#else - msi_for_each_desc(desc, cdev, MSI_DESC_ALL) { - switch (desc->msi_index) { -#endif - case 0: - ret = request_irq(desc->irq, pcie_dma_epf_wr0_msi, 0, - "pcie_dma_wr0", epfnv); - if (ret < 0) - dev_err(fdev, "failed to register wr0 irq\n"); - break; - case 1: - ret = request_irq(desc->irq, pcie_dma_epf_wr1_msi, 0, - "pcie_dma_wr1", epfnv); - if (ret < 0) - dev_err(fdev, "failed to register wr1 irq\n"); - break; - case 2: - ret = request_irq(desc->irq, pcie_dma_epf_wr2_msi, 0, - "pcie_dma_wr2", epfnv); - if (ret < 0) - dev_err(fdev, "failed to register wr2 irq\n"); - break; - case 3: - ret = request_irq(desc->irq, pcie_dma_epf_wr3_msi, 0, - "pcie_dma_wr3", epfnv); - if (ret < 0) - dev_err(fdev, "failed to register wr3 irq\n"); - break; - case 4: - ret = request_irq(desc->irq, pcie_dma_epf_rd0_msi, 0, - "pcie_dma_rd0", epfnv); - if (ret < 0) - dev_err(fdev, "failed to register rd0 irq\n"); - break; - case 5: - ret = request_irq(desc->irq, pcie_dma_epf_rd1_msi, 0, - "pcie_dma_rd1", epfnv); - if (ret < 0) - dev_err(fdev, "failed to register rd1 irq\n"); - break; - default: - dev_err(fdev, "Unknown MSI irq: %d\n", desc->irq); - continue; - } - } - - return 0; -} - -static void pcie_dma_epf_msi_deinit(struct pci_epf *epf) -{ - struct pcie_epf_dma *epfnv = epf_get_drvdata(epf); - struct pci_epc *epc = epf->epc; - struct device *cdev = epc->dev.parent; - struct device *fdev = &epf->dev; - struct msi_desc *desc; - - /* LL DMA in sanity test will not work without MSI for EP */ - if (!dev_get_msi_domain(cdev)) { - dev_info(fdev, "msi_domain absent, no interrupts\n"); - return; - } - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0) - for_each_msi_entry(desc, cdev) -#else - msi_for_each_desc(desc, cdev, MSI_DESC_ALL) -#endif - free_irq(desc->irq, epfnv); - - platform_msi_domain_free_irqs(cdev); -} - static int pcie_dma_epf_core_deinit(struct pci_epf *epf) { struct pcie_epf_dma *epfnv = epf_get_drvdata(epf); @@ -1574,7 +193,6 @@ static void pcie_dma_epf_unbind(struct pci_epf *epf) epf_bar0->rp_phy_addr = 0; tegra_pcie_edma_deinit(cookie); - pcie_dma_epf_msi_deinit(epf); pci_epc_stop(epc); lpci_epf_free_space(epf, epfnv->bar0_virt, BAR_0); } @@ -1588,7 +206,6 @@ static int pcie_dma_epf_bind(struct pci_epf *epf) struct platform_device *pdev = of_find_device_by_node(cdev->of_node); struct pci_epf_bar *epf_bar = &epf->bar[BAR_0]; struct pcie_epf_bar0 *epf_bar0; - struct resource *res; char *name; int ret, i; @@ -1612,56 +229,17 @@ static int pcie_dma_epf_bind(struct pci_epf *epf) epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu_dma"); - if (!res) { - dev_err(fdev, "missing atu_dma resource in DT\n"); - ret = PTR_ERR(res); - goto fail_atu_dma; - } - - epfnv->dma_base = devm_ioremap(fdev, res->start + DMA_OFFSET, - resource_size(res) - DMA_OFFSET); - if (IS_ERR(epfnv->dma_base)) { - ret = PTR_ERR(epfnv->dma_base); - dev_err(fdev, "dma region map failed: %d\n", ret); - goto fail_atu_dma; - } - - epfnv->irq = platform_get_irq_byname(pdev, "intr"); - if (!epfnv->irq) { - dev_err(fdev, "failed to get intr interrupt\n"); - ret = -ENODEV; - goto fail_atu_dma; - } - name = devm_kasprintf(fdev, GFP_KERNEL, "%s_epf_dma_test", pdev->name); if (!name) { ret = -ENOMEM; goto fail_atu_dma; } - ret = devm_request_threaded_irq(fdev, epfnv->irq, pcie_dma_epf_irq, - pcie_dma_epf_irq_handler, - IRQF_SHARED, - name, epfnv); - if (ret < 0) { - dev_err(fdev, "failed to request \"intr\" irq\n"); - goto fail_atu_dma; - } - - ret = pcie_dma_epf_msi_init(epf); - if (ret < 0) { - dev_err(fdev, "failed to init platform msi: %d\n", ret); - goto fail_atu_dma; - } - for (i = 0; i < DMA_WR_CHNL_NUM; i++) { - init_waitqueue_head(&epfnv->wr_wq[i]); init_waitqueue_head(&epfnv->edma.wr_wq[i]); } for (i = 0; i < DMA_RD_CHNL_NUM; i++) { - init_waitqueue_head(&epfnv->rd_wq[i]); init_waitqueue_head(&epfnv->edma.rd_wq[i]); }