PCI: Add T264 PCIe DMA support in PCIe EPF and client driver

Replace the exisiting T234 DMA APIs with common APIs and also add chip specific changes like BAR. Bug 4549851 Change-Id: I021e1e93a4fa4ff1d4429bd9db852e0e707ba879 Signed-off-by: Manikanta Maddireddy <mmaddireddy@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/3116906 Reviewed-by: Nagarjuna Kristam <nkristam@nvidia.com> Reviewed-by: Bibek Basu <bbasu@nvidia.com> GVS: buildbot_gerritrpt <buildbot_gerritrpt@nvidia.com>
2025-12-22 17:25:35 +03:00 · 2024-04-30 06:47:51 +00:00
parent 82e2f7b2d5
commit b0c580762b
3 changed files with 591 additions and 245 deletions
--- a/drivers/pci/endpoint/functions/pci-epf-dma-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-dma-test.c
@@ -14,10 +14,10 @@
 #include <linux/of_platform.h>
 #include <linux/pci-epc.h>
 #include <linux/pci-epf.h>
-#include <linux/pcie_dma.h>
 #include <linux/platform_device.h>
-#include <linux/kthread.h>
 #include <linux/tegra-pcie-edma-test-common.h>
+#include <linux/version.h>
+#include <soc/tegra/fuse-helper.h>
 #include "pci-epf-wrapper.h"

 static struct pcie_epf_dma *gepfnv;
@@ -28,43 +28,21 @@ struct pcie_epf_dma {
 	struct pci_epc *epc;
 	struct device *fdev;
 	struct device *cdev;
-	void *bar0_virt;
+	void *bar_virt;
 	struct dentry *debugfs;
-	void __iomem *dma_base;
+	void __iomem *dma_virt;
 	int irq;

+	u8 chip_id;
+
 	u32 dma_size;
 	u32 stress_count;
 	u32 async_count;

-	struct task_struct *wr0_task;
-	struct task_struct *wr1_task;
-	struct task_struct *wr2_task;
-	struct task_struct *wr3_task;
-	struct task_struct *rd0_task;
-	struct task_struct *rd1_task;
-	u8 task_done;
-	wait_queue_head_t task_wq;
-	void *cookie;
-
-	wait_queue_head_t wr_wq[DMA_WR_CHNL_NUM];
-	wait_queue_head_t rd_wq[DMA_RD_CHNL_NUM];
-	unsigned long wr_busy;
-	unsigned long rd_busy;
-	ktime_t wr_start_time[DMA_WR_CHNL_NUM];
-	ktime_t wr_end_time[DMA_WR_CHNL_NUM];
-	ktime_t rd_start_time[DMA_RD_CHNL_NUM];
-	ktime_t rd_end_time[DMA_RD_CHNL_NUM];
-	u32 wr_cnt[DMA_WR_CHNL_NUM + DMA_RD_CHNL_NUM];
-	u32 rd_cnt[DMA_WR_CHNL_NUM + DMA_RD_CHNL_NUM];
-	bool pcs[DMA_WR_CHNL_NUM + DMA_RD_CHNL_NUM];
-	bool async_dma;
-	ktime_t edma_start_time[DMA_WR_CHNL_NUM];
 	u64 tsz;
 	u32 edma_ch;
 	u32 prev_edma_ch;
 	u32 nents;
-	struct tegra_pcie_edma_desc *ll_desc;
 	struct edmalib_common edma;
 };

@@ -82,27 +60,54 @@ static void edma_lib_test_raise_irq(void *p)
 /* debugfs to perform eDMA lib transfers and do CRC check */
 static int edmalib_test(struct seq_file *s, void *data)
 {
-	struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)
-						dev_get_drvdata(s->private);
-	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *)
-						epfnv->bar0_virt;
+	struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)dev_get_drvdata(s->private);
+	struct pcie_epf_bar *epf_bar = (struct pcie_epf_bar *)epfnv->bar_virt;

-	if (!epf_bar0->rp_phy_addr) {
+	if (!epf_bar->rp_phy_addr) {
 		dev_err(epfnv->fdev, "RP DMA address is null\n");
 		return -1;
 	}

-	epfnv->edma.src_dma_addr = epf_bar0->ep_phy_addr + BAR0_DMA_BUF_OFFSET;
-	epfnv->edma.dst_dma_addr = epf_bar0->rp_phy_addr + BAR0_DMA_BUF_OFFSET;
+	epfnv->edma.src_dma_addr = epf_bar->ep_phy_addr + BAR0_DMA_BUF_OFFSET;
+	epfnv->edma.dst_dma_addr = epf_bar->rp_phy_addr + BAR0_DMA_BUF_OFFSET;
 	epfnv->edma.fdev = epfnv->fdev;
-	epfnv->edma.epf_bar0 = (struct pcie_epf_bar0 *)epfnv->bar0_virt;
-	epfnv->edma.src_virt = epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET;
-	epfnv->edma.dma_base = epfnv->dma_base;
+	epfnv->edma.cdev = epfnv->cdev;
+	epfnv->edma.epf_bar = (struct pcie_epf_bar *)epfnv->bar_virt;
+	epfnv->edma.src_virt = epfnv->bar_virt + BAR0_DMA_BUF_OFFSET;
+	epfnv->edma.dma_virt = epfnv->dma_virt;
+	epfnv->edma.dma_size = epfnv->dma_size;
+	epfnv->edma.stress_count = epfnv->stress_count;
+	epfnv->edma.edma_ch = epfnv->edma_ch;
+	epfnv->edma.nents = epfnv->nents;
+	epfnv->edma.priv = (void *)epfnv;
+	epfnv->edma.raise_irq = edma_lib_test_raise_irq;
+
+	return edmalib_common_test(&epfnv->edma);
+}
+
+/* debugfs to perform eDMA lib transfers and do CRC check */
+static int edmalib_read_test(struct seq_file *s, void *data)
+{
+	struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)dev_get_drvdata(s->private);
+	struct pcie_epf_bar *epf_bar = (struct pcie_epf_bar *)epfnv->bar_virt;
+
+	if (!epf_bar->rp_phy_addr) {
+		dev_err(epfnv->fdev, "RP DMA address is null\n");
+		return -1;
+	}
+
+	epfnv->edma.dst_dma_addr = epf_bar->ep_phy_addr + BAR0_DMA_BUF_OFFSET;
+	epfnv->edma.dst_dma_addr = epf_bar->rp_phy_addr + BAR0_DMA_BUF_OFFSET;
+	epfnv->edma.fdev = epfnv->fdev;
+	epfnv->edma.cdev = epfnv->cdev;
+
+	epfnv->edma.epf_bar = (struct pcie_epf_bar *)epfnv->bar_virt;
+	epfnv->edma.src_virt = epfnv->bar_virt + BAR0_DMA_BUF_OFFSET;
+	epfnv->edma.dma_virt = epfnv->dma_virt;
 	epfnv->edma.dma_size = epfnv->dma_size;
 	epfnv->edma.stress_count = epfnv->stress_count;
 	epfnv->edma.edma_ch = epfnv->edma_ch;
 	epfnv->edma.nents = epfnv->nents;
-	epfnv->edma.of_node = epfnv->cdev->of_node;
 	epfnv->edma.priv = (void *)epfnv;
 	epfnv->edma.raise_irq = edma_lib_test_raise_irq;

@@ -111,8 +116,9 @@ static int edmalib_test(struct seq_file *s, void *data)

 static void init_debugfs(struct pcie_epf_dma *epfnv)
 {
-	debugfs_create_devm_seqfile(epfnv->fdev, "edmalib_test", epfnv->debugfs,
-				    edmalib_test);
+	debugfs_create_devm_seqfile(epfnv->fdev, "edmalib_test", epfnv->debugfs, edmalib_test);
+	debugfs_create_devm_seqfile(epfnv->fdev, "edmalib_read_test", epfnv->debugfs,
+				    edmalib_read_test);

 	debugfs_create_u32("dma_size", 0644, epfnv->debugfs, &epfnv->dma_size);
 	epfnv->dma_size = SZ_1M;
@@ -132,16 +138,17 @@ static void init_debugfs(struct pcie_epf_dma *epfnv)
 	/* Set DMA_LL_DEFAULT_SIZE as default nents, Max NUM_EDMA_DESC */
 	epfnv->nents = DMA_LL_DEFAULT_SIZE;

-	debugfs_create_u32("stress_count", 0644, epfnv->debugfs,
-			   &epfnv->stress_count);
+	debugfs_create_u32("stress_count", 0644, epfnv->debugfs, &epfnv->stress_count);
 	epfnv->stress_count = DEFAULT_STRESS_COUNT;
 }

 static int pcie_dma_epf_core_init(struct pci_epf *epf)
 {
+	struct pcie_epf_dma *epfnv = epf_get_drvdata(epf);
 	struct pci_epc *epc = epf->epc;
 	struct device *fdev = &epf->dev;
 	struct pci_epf_bar *epf_bar;
+	enum pci_barno bar;
 	int ret;

 	ret = lpci_epc_write_header(epc, epf->func_no, epf->header);
@@ -150,15 +157,24 @@ static int pcie_dma_epf_core_init(struct pci_epf *epf)
 		return ret;
 	}

-	epf_bar = &epf->bar[BAR_0];
+	if (epfnv->chip_id == TEGRA234)
+		bar = BAR_0;
+	else
+		bar = BAR_1;
+
+	epf_bar = &epf->bar[bar];
 	ret = lpci_epc_set_bar(epc, epf->func_no, epf_bar);
 	if (ret < 0) {
 		dev_err(fdev, "PCIe set BAR0 failed: %d\n", ret);
 		return ret;
 	}

-	dev_info(fdev, "BAR0 phy_addr: %llx size: %lx\n",
-		 epf_bar->phys_addr, epf_bar->size);
+	dev_info(fdev, "BAR0 phy_addr: %llx size: %lx\n", epf_bar->phys_addr, epf_bar->size);
+
+	if (epf->msi_interrupts == 0) {
+		dev_err(fdev, "pci_epc_set_msi() failed: %d\n", epf->msi_interrupts);
+		epf->msi_interrupts = 16;
+	}

 	ret = lpci_epc_set_msi(epc, epf->func_no, epf->msi_interrupts);
 	if (ret) {
@@ -166,6 +182,22 @@ static int pcie_dma_epf_core_init(struct pci_epf *epf)
 		return ret;
 	}

+	if (epfnv->chip_id == TEGRA234)
+		return 0;
+
+	/* Expose MSI address as BAR2 to allow RP to send MSI to EP. */
+	epf_bar = &epf->bar[BAR_2];
+	epf_bar[bar].phys_addr = gepfnv->edma.msi_addr & ~(SZ_32M - 1);
+	epf_bar[bar].addr = NULL;
+	epf_bar[bar].size = SZ_32M;
+	epf_bar[bar].barno = BAR_2;
+	epf_bar[bar].flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+	ret = lpci_epc_set_bar(epc, epf->func_no, epf_bar);
+	if (ret < 0) {
+		dev_err(fdev, "PCIe set BAR2 failed: %d\n", ret);
+		return ret;
+	}
+
 	return 0;
 }

@@ -174,14 +206,22 @@ static int pcie_dma_epf_core_deinit(struct pci_epf *epf)
 {
 	struct pcie_epf_dma *epfnv = epf_get_drvdata(epf);
 	void *cookie = epfnv->edma.cookie;
-	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) epfnv->bar0_virt;
+	struct pcie_epf_bar *epf_bar_virt = (struct pcie_epf_bar *)epfnv->bar_virt;
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_bar *epf_bar = &epf->bar[BAR_0];
+	struct pci_epf_bar *epf_bar;
+	enum pci_barno bar;

+	if (epfnv->chip_id == TEGRA234)
+		bar = BAR_0;
+	else
+		bar = BAR_1;
+	epf_bar = &epf->bar[bar];
 	epfnv->edma.cookie = NULL;
-	epf_bar0->rp_phy_addr = 0;
-	tegra_pcie_edma_deinit(cookie);
+	epf_bar_virt->rp_phy_addr = 0;
+	tegra_pcie_dma_deinit(&cookie);
 	lpci_epc_clear_bar(epc, epf->func_no, epf_bar);
+	if (epfnv->chip_id == TEGRA264)
+		lpci_epc_clear_bar(epc, epf->func_no, &epf->bar[BAR_2]);

 	return 0;
 }
@@ -192,16 +232,54 @@ static void pcie_dma_epf_unbind(struct pci_epf *epf)
 	struct pcie_epf_dma *epfnv = epf_get_drvdata(epf);
 	struct pci_epc *epc = epf->epc;
 	void *cookie = epfnv->edma.cookie;
-	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) epfnv->bar0_virt;
+	struct pcie_epf_bar *epf_bar = (struct pcie_epf_bar *)epfnv->bar_virt;
+	struct device *cdev = epc->dev.parent;
+	struct platform_device *pdev = of_find_device_by_node(cdev->of_node);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
+	struct msi_desc *desc;
+#endif
+	enum pci_barno bar;
+	u32 irq;

 	debugfs_remove_recursive(epfnv->debugfs);

 	epfnv->edma.cookie = NULL;
-	epf_bar0->rp_phy_addr = 0;
-	tegra_pcie_edma_deinit(cookie);
+	epf_bar->rp_phy_addr = 0;
+	tegra_pcie_dma_deinit(&cookie);

+	if (epfnv->chip_id == TEGRA264) {
+		platform_msi_domain_free_irqs(&pdev->dev);
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+		irq = msi_get_virq(&pdev->dev, 0);
+#else
+		for_each_msi_entry(desc, cdev) {
+			if (desc->platform.msi_index == 0)
+				irq = desc->irq;
+		}
+#endif
+		free_irq(irq, epfnv);
+	}
 	pci_epc_stop(epc);
-	lpci_epf_free_space(epf, epfnv->bar0_virt, BAR_0);
+	if (epfnv->chip_id == TEGRA234)
+		bar = BAR_0;
+	else
+		bar = BAR_1;
+	lpci_epf_free_space(epf, epfnv->bar_virt, bar);
+}
+
+static void pcie_dma_epf_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
+{
+	if (gepfnv->edma.msi_addr == 0) {
+		gepfnv->edma.msi_addr = msg->address_hi;
+		gepfnv->edma.msi_addr <<= 32;
+		gepfnv->edma.msi_addr |= msg->address_lo;
+		gepfnv->edma.msi_data = msg->data + 1;
+	}
+}
+
+static irqreturn_t pcie_dma_epf_irq(int irq, void *arg)
+{
+	return IRQ_HANDLED;
 }

 static int pcie_dma_epf_bind(struct pci_epf *epf)
@@ -212,11 +290,26 @@ static int pcie_dma_epf_bind(struct pci_epf *epf)
 	struct device *fdev = &epf->dev;
 	struct device *cdev = epc->dev.parent;
 	struct platform_device *pdev = of_find_device_by_node(cdev->of_node);
-	struct pci_epf_bar *epf_bar = &epf->bar[BAR_0];
-	struct pcie_epf_bar0 *epf_bar0;
+	struct pcie_epf_bar *epf_bar_virt;
+	struct pci_epf_bar *epf_bar;
+	struct irq_domain *domain;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)
+	struct msi_desc *desc;
+#endif
+	enum pci_barno bar;
 	char *name;
 	int ret, i;
+	u32 irq;

+	epfnv->chip_id = __tegra_get_chip_id();
+	if (epfnv->chip_id == TEGRA234) {
+		bar = BAR_0;
+		epfnv->edma.chip_id = NVPCIE_DMA_SOC_T234;
+	} else {
+		bar = BAR_1;
+		epfnv->edma.chip_id = NVPCIE_DMA_SOC_T264;
+	}
+	epf_bar = &epf->bar[bar];
 	epfnv->fdev = fdev;
 	epfnv->cdev = cdev;
 	epfnv->epf = epf;
@@ -229,25 +322,22 @@ static int pcie_dma_epf_bind(struct pci_epf *epf)
 	}

 #if defined(NV_PCI_EPF_ALLOC_SPACE_HAS_EPC_FEATURES_ARG) /* Linux v6.9 */
-	epfnv->bar0_virt = lpci_epf_alloc_space(epf, BAR0_SIZE, BAR_0,
-						epc_features);
+	epfnv->bar_virt = lpci_epf_alloc_space(epf, BAR0_SIZE, BAR_0, epc_features);
 #else
-	epfnv->bar0_virt = lpci_epf_alloc_space(epf, BAR0_SIZE, BAR_0,
-						epc_features->align);
+	epfnv->bar_virt = lpci_epf_alloc_space(epf, BAR0_SIZE, BAR_0, epc_features->align);
 #endif
-	if (!epfnv->bar0_virt) {
+	if (!epfnv->bar_virt) {
 		dev_err(fdev, "Failed to allocate memory for BAR0\n");
 		return -ENOMEM;
 	}
-	get_random_bytes(epfnv->bar0_virt, BAR0_SIZE);
-	memset(epfnv->bar0_virt, 0, BAR0_HEADER_SIZE);
+	get_random_bytes(epfnv->bar_virt, BAR0_SIZE);
+	memset(epfnv->bar_virt, 0, BAR0_HEADER_SIZE);

 	/* Update BAR header with EP DMA PHY addr */
-	epf_bar0 = (struct pcie_epf_bar0 *)epfnv->bar0_virt;
-	epf_bar0->ep_phy_addr = epf_bar->phys_addr;
+	epf_bar_virt = (struct pcie_epf_bar *)epfnv->bar_virt;
+	epf_bar_virt->ep_phy_addr = epf_bar->phys_addr;
 	/* Set BAR0 mem type as 64-bit */
-	epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64 |
-			PCI_BASE_ADDRESS_MEM_PREFETCH;
+	epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64 | PCI_BASE_ADDRESS_MEM_PREFETCH;

 	name = devm_kasprintf(fdev, GFP_KERNEL, "%s_epf_dma_test", pdev->name);
 	if (!name) {
@@ -255,21 +345,73 @@ static int pcie_dma_epf_bind(struct pci_epf *epf)
 		goto fail_atu_dma;
 	}

-	for (i = 0; i < DMA_WR_CHNL_NUM; i++) {
+	for (i = 0; i < TEGRA_PCIE_DMA_WRITE; i++)
 		init_waitqueue_head(&epfnv->edma.wr_wq[i]);
-	}

-	for (i = 0; i < DMA_RD_CHNL_NUM; i++) {
-		init_waitqueue_head(&epfnv->edma.rd_wq[i]);
+	if (epfnv->chip_id == TEGRA234) {
+		domain = dev_get_msi_domain(&pdev->dev);
+		if (!domain) {
+			dev_err(fdev, "failed to get MSI domain\n");
+			ret = -ENOMEM;
+			goto fail_kasnprintf;
+		}
+
+		ret = platform_msi_domain_alloc_irqs(&pdev->dev, 2, pcie_dma_epf_write_msi_msg);
+		if (ret < 0) {
+			dev_err(fdev, "failed to allocate MSIs: %d\n", ret);
+			goto fail_kasnprintf;
+		}
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)
+		epfnv->edma.msi_irq = msi_get_virq(&pdev->dev, 1);
+		irq = msi_get_virq(&pdev->dev, 0);
+#else
+		for_each_msi_entry(desc, cdev) {
+			if (desc->platform.msi_index == 0)
+				irq = desc->irq;
+			else if (desc->platform.msi_index == 1)
+				epfnv->edma.msi_irq = desc->irq;
+		}
+#endif
+
+		ret = request_irq(irq, pcie_dma_epf_irq, IRQF_SHARED, "pcie_dma_epf_isr", epfnv);
+		if (ret < 0) {
+			dev_err(fdev, "failed to request irq: %d\n", ret);
+			goto fail_msi_alloc;
+		}
 	}

 	epfnv->debugfs = debugfs_create_dir(name, NULL);
 	init_debugfs(epfnv);

+	epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
+	if (!epc_features) {
+		dev_err(fdev, "epc_features not implemented\n");
+		ret = -EOPNOTSUPP;
+		goto fail_get_features;
+	}
+
+	if (!epc_features->core_init_notifier) {
+		ret = pcie_dma_epf_core_init(epf);
+		if (ret) {
+			dev_err(fdev, "EPF core init failed with err: %d\n", ret);
+			goto fail_core_init;
+		}
+	}
+
 	return 0;

+fail_core_init:
+fail_get_features:
+	debugfs_remove_recursive(epfnv->debugfs);
+	if (epfnv->chip_id == TEGRA264)
+		free_irq(irq, epfnv);
+fail_msi_alloc:
+	if (epfnv->chip_id == TEGRA264)
+		platform_msi_domain_free_irqs(&pdev->dev);
+fail_kasnprintf:
+	devm_kfree(fdev, name);
 fail_atu_dma:
-	lpci_epf_free_space(epf, epfnv->bar0_virt, BAR_0);
+	lpci_epf_free_space(epf, epfnv->bar_virt, bar);

 	return ret;
 }
@@ -301,10 +443,10 @@ static int pcie_dma_epf_probe(struct pci_epf *epf)
 	if (!epfnv)
 		return -ENOMEM;

-	epfnv->edma.ll_desc = devm_kzalloc(dev, sizeof(*epfnv->ll_desc) * NUM_EDMA_DESC,
+	epfnv->edma.ll_desc = devm_kzalloc(dev, sizeof(*epfnv->edma.ll_desc) * NUM_EDMA_DESC,
 					   GFP_KERNEL);
-	gepfnv = epfnv;
 	epf_set_drvdata(epf, epfnv);
+	gepfnv = epfnv;

 	epf->event_ops = &pci_epf_dma_test_event_ops;