PCI: Add data integrity check for PCIe DMA

Add CRC check after PCIe DMA to verify data integrity.

Bug 3636902
Bug 3868928

Change-Id: If497ce769571c6c837acbfbbb64b2242dfabef26
Signed-off-by: Manikanta Maddireddy <mmaddireddy@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2718787
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2720336
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/2815927
Reviewed-by: Nagarjuna Kristam <nkristam@nvidia.com>
Reviewed-by: Bitan Biswas <bbiswas@nvidia.com>
Tested-by: Nagarjuna Kristam <nkristam@nvidia.com>
GVS: Gerrit_Virtual_Submit <buildbot_gerritrpt@nvidia.com>

drivers/misc/tegra-pcie-dma-test.c

@@ -85,11 +85,25 @@ static irqreturn_t ep_isr(int irq, void *arg)
 			       ep->bar0_virt + BAR0_MSI_OFFSET);
 		}
 	}
+#else
+	struct ep_pvt *ep = (struct ep_pvt *)arg;
+	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *)ep->bar0_virt;
+
+	epf_bar0->wr_data[0].crc = crc32_le(~0,	ep->dma_virt + BAR0_DMA_BUF_OFFSET,
+					    epf_bar0->wr_data[0].size);
 #endif
 
 	return IRQ_HANDLED;
 }
 
+static void tegra_pcie_dma_raise_irq(void *p)
+{
+	struct ep_pvt *ep = (struct ep_pvt *)p;
+	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *)ep->bar0_virt;
+
+	writel(epf_bar0->msi_data[0], ep->bar0_virt + BAR0_MSI_OFFSET);
+}
+
 extern struct device *naga_pci[];
 struct edma_desc {
 	dma_addr_t src;
@@ -132,9 +146,11 @@ static int edmalib_test(struct seq_file *s, void *data)
 	ep->edma.src_dma_addr = rp_dma_addr;
 	ep->edma.src_virt = ep->dma_virt + SZ_128M + SZ_1M;
 	ep->edma.fdev = &ep->pdev->dev;
-	ep->edma.bar0_virt = ep->bar0_virt;
+	ep->edma.epf_bar0 = (struct pcie_epf_bar0 *)ep->bar0_virt;
 	ep->edma.bar0_phy = ep->bar0_phy;
 	ep->edma.dma_base = ep->dma_base;
+	ep->edma.priv = (void *)ep;
+	ep->edma.raise_irq = tegra_pcie_dma_raise_irq;
 
 	if (REMOTE_EDMA_TEST_EN) {
 		ep->edma.dst_dma_addr = ep_dma_addr;
@@ -229,12 +245,14 @@ static int ep_test_dma_probe(struct pci_dev *pdev,
 		goto fail_region_remap;
 	}
 
-	if (pci_enable_msi(pdev) < 0) {
+	ret = pci_alloc_irq_vectors(pdev, 2, 2, PCI_IRQ_MSI);
+	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to enable MSI interrupt\n");
 		ret = -ENODEV;
 		goto fail_region_remap;
 	}
-	ret = request_irq(pdev->irq, (irq_handler_t)ep_isr, IRQF_SHARED,
+
+	ret = request_irq(pci_irq_vector(pdev, 1), ep_isr, IRQF_SHARED,
 			  "pcie_ep_isr", ep);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register isr\n");
@@ -305,7 +323,7 @@ static int ep_test_dma_probe(struct pci_dev *pdev,
 	}
 	pci_read_config_dword(pdev, pdev->msi_cap + PCI_MSI_ADDRESS_LO, &val);
 	ep->msi_addr = (ep->msi_addr << 32) | val;
-	ep->msi_irq = pdev->irq;
+	ep->msi_irq = pci_irq_vector(pdev, 0);
 	ep->dma_phy_base = pci_resource_start(pdev, 4);
 	ep->dma_phy_size = pci_resource_len(pdev, 4);
 
@@ -330,9 +348,9 @@ static int ep_test_dma_probe(struct pci_dev *pdev,
 fail_name:
 	dma_free_coherent(&pdev->dev, BAR0_SIZE, ep->dma_virt, ep->dma_phy);
 fail_dma_alloc:
-	free_irq(pdev->irq, ep);
+	free_irq(pci_irq_vector(pdev, 1), ep);
 fail_isr:
-	pci_disable_msi(pdev);
+	pci_free_irq_vectors(pdev);
 fail_region_remap:
 	pci_release_regions(pdev);
 fail_region_request:
@@ -348,8 +366,8 @@ static void ep_test_dma_remove(struct pci_dev *pdev)
 	debugfs_remove_recursive(ep->debugfs);
 	tegra_pcie_edma_deinit(ep->cookie);
 	dma_free_coherent(&pdev->dev, BAR0_SIZE, ep->dma_virt, ep->dma_phy);
-	free_irq(pdev->irq, ep);
+	free_irq(pci_irq_vector(pdev, 1), ep);
-	pci_disable_msi(pdev);
+	pci_free_irq_vectors(pdev);
 	pci_release_regions(pdev);
 	pci_clear_master(pdev);
 }

drivers/pci/endpoint/functions/pci-epf-dma-test.c

@@ -23,6 +23,8 @@ static struct pcie_epf_dma *gepfnv;
 
 struct pcie_epf_dma {
 	struct pci_epf_header header;
+	struct pci_epf *epf;
+	struct pci_epc *epc;
 	struct device *fdev;
 	struct device *cdev;
 	void *bar0_virt;
@@ -74,10 +76,10 @@ struct edma_desc {
 static irqreturn_t pcie_dma_epf_wr0_msi(int irq, void *arg)
 {
 	struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)arg;
-	int bit = 0;
+	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *)epfnv->bar0_virt;
 
-	epfnv->wr_busy &= ~BIT(bit);
+	epf_bar0->wr_data[0].crc = crc32_le(~0, epfnv->bar0_virt + SZ_128M + BAR0_DMA_BUF_OFFSET,
-	wake_up(&epfnv->wr_wq[bit]);
+					    epf_bar0->wr_data[0].size);
 
 	return IRQ_HANDLED;
 }
@@ -881,6 +883,13 @@ fail:
 	return 0;
 }
 
+static void edma_lib_test_raise_irq(void *p)
+{
+	struct pcie_epf_dma *epfnv = (struct pcie_epf_dma *)p;
+
+	lpci_epc_raise_irq(epfnv->epc, epfnv->epf->func_no, PCI_EPC_IRQ_MSI, 1);
+}
+
 /* debugfs to perform eDMA lib transfers and do CRC check */
 static int edmalib_test(struct seq_file *s, void *data)
 {
@@ -897,7 +906,7 @@ static int edmalib_test(struct seq_file *s, void *data)
 	epfnv->edma.src_dma_addr = epf_bar0->ep_phy_addr + BAR0_DMA_BUF_OFFSET;
 	epfnv->edma.dst_dma_addr = epf_bar0->rp_phy_addr + BAR0_DMA_BUF_OFFSET;
 	epfnv->edma.fdev = epfnv->fdev;
-	epfnv->edma.bar0_virt = epfnv->bar0_virt;
+	epfnv->edma.epf_bar0 = (struct pcie_epf_bar0 *)epfnv->bar0_virt;
 	epfnv->edma.src_virt = epfnv->bar0_virt + BAR0_DMA_BUF_OFFSET;
 	epfnv->edma.dma_base = epfnv->dma_base;
 	epfnv->edma.dma_size = epfnv->dma_size;
@@ -905,6 +914,8 @@ static int edmalib_test(struct seq_file *s, void *data)
 	epfnv->edma.edma_ch = epfnv->edma_ch;
 	epfnv->edma.nents = epfnv->nents;
 	epfnv->edma.of_node = epfnv->cdev->of_node;
+	epfnv->edma.priv = (void *)epfnv;
+	epfnv->edma.raise_irq = edma_lib_test_raise_irq;
 
 	return edmalib_common_test(&epfnv->edma);
 }
@@ -1554,7 +1565,6 @@ static void pcie_dma_epf_unbind(struct pci_epf *epf)
 {
 	struct pcie_epf_dma *epfnv = epf_get_drvdata(epf);
 	struct pci_epc *epc = epf->epc;
-	struct pci_epf_bar *epf_bar = &epf->bar[BAR_0];
 	void *cookie = epfnv->edma.cookie;
 	struct pcie_epf_bar0 *epf_bar0 = (struct pcie_epf_bar0 *) epfnv->bar0_virt;
 
@@ -1584,6 +1594,8 @@ static int pcie_dma_epf_bind(struct pci_epf *epf)
 
 	epfnv->fdev = fdev;
 	epfnv->cdev = cdev;
+	epfnv->epf = epf;
+	epfnv->epc = epc;
 
 	epfnv->bar0_virt = lpci_epf_alloc_space(epf, BAR0_SIZE, BAR_0, SZ_64K);
 	if (!epfnv->bar0_virt) {

include/linux/tegra-pcie-edma-test-common.h

@@ -14,6 +14,7 @@
 
 #define EDMA_ABORT_TEST_EN	(edma->edma_ch & 0x40000000)
 #define EDMA_STOP_TEST_EN	(edma->edma_ch & 0x20000000)
+#define EDMA_CRC_TEST_EN	(edma->edma_ch & 0x10000000)
 #define IS_EDMA_CH_ENABLED(i)	(edma->edma_ch & ((BIT(i) << 4)))
 #define IS_EDMA_CH_ASYNC(i)	(edma->edma_ch & BIT(i))
 #define REMOTE_EDMA_TEST_EN	(edma->edma_ch & 0x80000000)
@@ -26,7 +27,9 @@
 
 struct edmalib_common {
 	struct device *fdev;
-	void *bar0_virt;
+	void (*raise_irq)(void *p);
+	void *priv;
+	struct pcie_epf_bar0 *epf_bar0;
 	void *src_virt;
 	void __iomem *dma_base;
 	u32 dma_size;
@@ -114,6 +117,7 @@ static int edmalib_common_test(struct edmalib_common *edma)
 	char *xfer_str[2] = {"WR", "RD"};
 	u32 l_r;
 	char *l_r_str[2] = {"local", "remote"};
+	struct pcie_epf_bar0 *epf_bar0 = edma->epf_bar0;
 
 	if (!edma->stress_count) {
 		tegra_pcie_edma_deinit(edma->cookie);
@@ -129,6 +133,16 @@ static int edmalib_common_test(struct edmalib_common *edma)
 		edma->edma_ch |= 0xF5;
 	}
 
+	if (EDMA_CRC_TEST_EN) {
+		/* 4 channels in sync mode */
+		edma->edma_ch = (EDMA_CRC_TEST_EN | 0xF0);
+		/* Single SZ_4K packet on each channel, so total SZ_16K of data */
+		edma->stress_count  = 1;
+		edma->dma_size = SZ_4K;
+		edma->nents = nents = 4;
+		epf_bar0->wr_data[0].size = edma->dma_size * edma->nents;
+	}
+
 	if (edma->cookie && edma->prev_edma_ch != edma->edma_ch) {
 		edma->st_as_ch = -1;
 		dev_info(edma->fdev, "edma_ch changed from 0x%x != 0x%x, deinit\n",
@@ -291,6 +305,19 @@ static int edmalib_common_test(struct edmalib_common *edma)
 				edma->tsz, diff, EDMA_PERF);
 		}
 	}
+
+	if (EDMA_CRC_TEST_EN && !REMOTE_EDMA_TEST_EN) {
+		u32 crc;
+
+		edma->raise_irq(edma->priv);
+		crc = crc32_le(~0, edma->src_virt, epf_bar0->wr_data[0].size);
+		msleep(100);
+		if (crc != epf_bar0->wr_data[0].crc)
+			dev_err(edma->fdev, "CRC check failed, LCRC: 0x%x RCRC: 0x%x\n",
+				crc, epf_bar0->wr_data[0].crc);
+		else
+			dev_err(edma->fdev, "CRC check pass\n");
+	}
 	dev_info(edma->fdev, "%s: EDMA LIB submit done\n", __func__);
 
 	return 0;