mirror of
git://nv-tegra.nvidia.com/linux-nv-oot.git
synced 2025-12-22 09:11:26 +03:00
pci: epf: Add dma test function driver
Add DMA test function driver from Nvidia repro Bug 3583632 Change-Id: I603d16f74cba2b4752ab31af59b12e7c784f042e Signed-off-by: Nagarjuna Kristam <nkristam@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/2736247 Reviewed-by: Manikanta Maddireddy <mmaddireddy@nvidia.com> Reviewed-by: Laxman Dewangan <ldewangan@nvidia.com> GVS: Gerrit_Virtual_Submit
This commit is contained in:
Nagarjuna Kristam
committed by
mobile promotions
mobile promotions
parent
0d83600487
commit
87f4a51862
@@ -2,3 +2,4 @@
|
|||||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||||
|
|
||||||
obj-m += controller/
|
obj-m += controller/
|
||||||
|
obj-m += endpoint/
|
||||||
|
|||||||
4
drivers/pci/endpoint/Makefile
Normal file
4
drivers/pci/endpoint/Makefile
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0
|
||||||
|
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||||
|
|
||||||
|
obj-m += functions/
|
||||||
4
drivers/pci/endpoint/functions/Makefile
Normal file
4
drivers/pci/endpoint/functions/Makefile
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0
|
||||||
|
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||||
|
|
||||||
|
obj-m += pci-epf-dma-test.o
|
||||||
1766
drivers/pci/endpoint/functions/pci-epf-dma-test.c
Normal file
1766
drivers/pci/endpoint/functions/pci-epf-dma-test.c
Normal file
File diff suppressed because it is too large
Load Diff
33
drivers/pci/endpoint/functions/pci-epf-wrapper.h
Normal file
33
drivers/pci/endpoint/functions/pci-epf-wrapper.h
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
/* SPDX-License-Identifier: GPL-2.0+ */
|
||||||
|
/*
|
||||||
|
* PCIe EDMA Framework
|
||||||
|
*
|
||||||
|
* Copyright (C) 2022 NVIDIA Corporation. All rights reserved.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef PCI_EPF_WRAPPER_H
|
||||||
|
#define PCI_EPF_WRAPPER_H
|
||||||
|
|
||||||
|
#include <linux/version.h>
|
||||||
|
|
||||||
|
#if (LINUX_VERSION_CODE > KERNEL_VERSION(5, 14, 0))
|
||||||
|
#define lpci_epc_write_header(A, B, C) pci_epc_write_header(A, B, 0, C)
|
||||||
|
#define lpci_epc_raise_irq(A, B, C, D) pci_epc_raise_irq(A, B, 0, C, D)
|
||||||
|
#define lpci_epc_clear_bar(A, B, C) pci_epc_clear_bar(A, B, 0, C)
|
||||||
|
#define lpci_epc_set_msi(A, B, C) pci_epc_set_msi(A, B, 0, C)
|
||||||
|
#define lpci_epc_set_bar(A, B, C) pci_epc_set_bar(A, B, 0, C)
|
||||||
|
#define lpci_epc_unmap_addr(A, B, C) pci_epc_unmap_addr(A, B, 0, C)
|
||||||
|
#define lpci_epf_free_space(A, B, C) pci_epf_free_space(A, B, C, PRIMARY_INTERFACE)
|
||||||
|
#define lpci_epf_alloc_space(A, B, C, D) pci_epf_alloc_space(A, B, C, D, PRIMARY_INTERFACE)
|
||||||
|
#else
|
||||||
|
#define lpci_epc_write_header(A, B, C) pci_epc_write_header(A, B, C)
|
||||||
|
#define lpci_epc_raise_irq(A, B, C, D) pci_epc_raise_irq(A, B, C, D)
|
||||||
|
#define lpci_epc_clear_bar(A, B, C) pci_epc_clear_bar(A, B, C)
|
||||||
|
#define lpci_epc_set_msi(A, B, C) pci_epc_set_msi(A, B, C)
|
||||||
|
#define lpci_epc_set_bar(A, B, C) pci_epc_set_bar(A, B, C)
|
||||||
|
#define lpci_epc_unmap_addr(A, B, C) pci_epc_unmap_addr(A, B, C)
|
||||||
|
#define lpci_epf_free_space(A, B, C) pci_epf_free_space(A, B, C)
|
||||||
|
#define lpci_epf_alloc_space(A, B, C, D) pci_epf_alloc_space(A, B, C, D)
|
||||||
|
#endif /* LINUX_VERSION_CODE */
|
||||||
|
|
||||||
|
#endif /* PCI_EPF_WRAPPER_H */
|
||||||
186
include/linux/pcie_dma.h
Normal file
186
include/linux/pcie_dma.h
Normal file
@@ -0,0 +1,186 @@
|
|||||||
|
/* SPDX-License-Identifier: GPL-2.0 */
|
||||||
|
/*
|
||||||
|
* PCIe DMA test framework for Tegra PCIe
|
||||||
|
*
|
||||||
|
* Copyright (C) 2021-2022 NVIDIA Corporation. All rights reserved.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef PCIE_DMA_H
|
||||||
|
#define PCIE_DMA_H
|
||||||
|
|
||||||
|
/* Update DMA_DD_BUF_SIZE and DMA_LL_BUF_SIZE when changing BAR0_SIZE */
|
||||||
|
#define BAR0_SIZE SZ_256M
|
||||||
|
|
||||||
|
/* Header includes RP/EP DMA addresses, EP MSI, LL, etc. */
|
||||||
|
#define BAR0_HEADER_OFFSET 0x0
|
||||||
|
#define BAR0_HEADER_SIZE SZ_1M
|
||||||
|
#define DMA_LL_OFFSET SZ_4K
|
||||||
|
#define DMA_LL_SIZE SZ_4K /* 4K size of LL serve 170 desc */
|
||||||
|
#define DMA_LL_WR_OFFSET(i) (DMA_LL_OFFSET + ((i) * DMA_LL_SIZE))
|
||||||
|
#define DMA_LL_RD_OFFSET(i) (DMA_LL_WR_OFFSET(4) + ((i) * DMA_LL_SIZE))
|
||||||
|
#define DMA_LL_MIN_SIZE 1
|
||||||
|
#define DMA_LL_DEFAULT_SIZE 8
|
||||||
|
#define DMA_ASYNC_LL_SIZE 160
|
||||||
|
|
||||||
|
#define BAR0_MSI_OFFSET SZ_64K
|
||||||
|
|
||||||
|
/* DMA'able memory range */
|
||||||
|
#define BAR0_DMA_BUF_OFFSET SZ_1M
|
||||||
|
#define BAR0_DMA_BUF_SIZE (BAR0_SIZE - SZ_1M)
|
||||||
|
#define DMA_DD_BUF_SIZE SZ_32M
|
||||||
|
#define DMA_LL_BUF_SIZE SZ_4M
|
||||||
|
|
||||||
|
/* Each DMA LL channel gets DMA_DD_BUF_SIZE and each desc DMA_LL_BUF_SIZE */
|
||||||
|
#define DMA_LL_WR_BUF(i) (BAR0_DMA_BUF_OFFSET + (i) * DMA_DD_BUF_SIZE)
|
||||||
|
#define DMA_LL_RD_BUF(i) (DMA_LL_WR_BUF(4) + (i) * DMA_DD_BUF_SIZE)
|
||||||
|
|
||||||
|
#define DEFAULT_STRESS_COUNT 10
|
||||||
|
|
||||||
|
#define MAX_DMA_ELE_SIZE SZ_16M
|
||||||
|
|
||||||
|
/* DMA base offset starts at 0x20000 from ATU_DMA base */
|
||||||
|
#define DMA_OFFSET 0x20000
|
||||||
|
|
||||||
|
#define DMA_RD_CHNL_NUM 2
|
||||||
|
#define DMA_RD_CHNL_MASK 0x3
|
||||||
|
#define DMA_WR_CHNL_NUM 4
|
||||||
|
#define DMA_WR_CHNL_MASK 0xf
|
||||||
|
|
||||||
|
/* DMA common registers */
|
||||||
|
#define DMA_WRITE_ENGINE_EN_OFF 0xC
|
||||||
|
#define WRITE_ENABLE BIT(0)
|
||||||
|
#define WRITE_DISABLE 0x0
|
||||||
|
|
||||||
|
#define DMA_WRITE_DOORBELL_OFF 0x10
|
||||||
|
#define DMA_WRITE_DOORBELL_OFF_WR_STOP BIT(31)
|
||||||
|
|
||||||
|
#define DMA_READ_ENGINE_EN_OFF 0x2C
|
||||||
|
#define READ_ENABLE BIT(0)
|
||||||
|
#define READ_DISABLE 0x0
|
||||||
|
|
||||||
|
#define DMA_READ_DOORBELL_OFF 0x30
|
||||||
|
#define DMA_READ_DOORBELL_OFF_RD_STOP BIT(31)
|
||||||
|
|
||||||
|
#define DMA_WRITE_INT_STATUS_OFF 0x4C
|
||||||
|
#define DMA_WRITE_INT_MASK_OFF 0x54
|
||||||
|
#define DMA_WRITE_INT_CLEAR_OFF 0x58
|
||||||
|
#define DMA_WRITE_INT_DONE_MASK 0xF
|
||||||
|
#define DMA_WRITE_INT_ABORT_MASK 0xF0000
|
||||||
|
|
||||||
|
#define DMA_WRITE_ERR_STATUS_OFF 0x5C
|
||||||
|
|
||||||
|
#define DMA_WRITE_DONE_IMWR_LOW_OFF 0x60
|
||||||
|
#define DMA_WRITE_DONE_IMWR_HIGH_OFF 0x64
|
||||||
|
#define DMA_WRITE_ABORT_IMWR_LOW_OFF 0x68
|
||||||
|
#define DMA_WRITE_ABORT_IMWR_HIGH_OFF 0x6C
|
||||||
|
|
||||||
|
#define DMA_WRITE_IMWR_DATA_OFF_BASE 0x70
|
||||||
|
|
||||||
|
#define DMA_READ_INT_STATUS_OFF 0xA0
|
||||||
|
#define DMA_READ_INT_MASK_OFF 0xA8
|
||||||
|
#define DMA_READ_INT_CLEAR_OFF 0xAC
|
||||||
|
#define DMA_READ_INT_DONE_MASK 0xF
|
||||||
|
#define DMA_READ_INT_ABORT_MASK 0xF0000
|
||||||
|
|
||||||
|
#define DMA_READ_DONE_IMWR_LOW_OFF 0xCC
|
||||||
|
#define DMA_READ_DONE_IMWR_HIGH_OFF 0xD0
|
||||||
|
#define DMA_READ_ABORT_IMWR_LOW_OFF 0xD4
|
||||||
|
#define DMA_READ_ABORT_IMWR_HIGH_OFF 0xD8
|
||||||
|
|
||||||
|
#define DMA_READ_IMWR_DATA_OFF_BASE 0xDC
|
||||||
|
|
||||||
|
/* DMA channel specific registers */
|
||||||
|
#define DMA_CH_CONTROL1_OFF_WRCH 0x0
|
||||||
|
#define DMA_CH_CONTROL1_OFF_WRCH_LLE BIT(9)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_WRCH_CCS BIT(8)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_WRCH_RIE BIT(4)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_WRCH_LIE BIT(3)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_WRCH_LLP BIT(2)
|
||||||
|
#define DMA_TRANSFER_SIZE_OFF_WRCH 0x8
|
||||||
|
#define DMA_SAR_LOW_OFF_WRCH 0xC
|
||||||
|
#define DMA_SAR_HIGH_OFF_WRCH 0x10
|
||||||
|
#define DMA_DAR_LOW_OFF_WRCH 0x14
|
||||||
|
#define DMA_DAR_HIGH_OFF_WRCH 0x18
|
||||||
|
#define DMA_LLP_LOW_OFF_WRCH 0x1C
|
||||||
|
#define DMA_LLP_HIGH_OFF_WRCH 0x20
|
||||||
|
|
||||||
|
#define DMA_CH_CONTROL1_OFF_RDCH 0x100
|
||||||
|
#define DMA_CH_CONTROL1_OFF_RDCH_LLE BIT(9)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_RDCH_CCS BIT(8)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_RDCH_RIE BIT(4)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_RDCH_LIE BIT(3)
|
||||||
|
#define DMA_CH_CONTROL1_OFF_RDCH_LLP BIT(2)
|
||||||
|
#define DMA_TRANSFER_SIZE_OFF_RDCH 0x108
|
||||||
|
#define DMA_SAR_LOW_OFF_RDCH 0x10C
|
||||||
|
#define DMA_SAR_HIGH_OFF_RDCH 0x110
|
||||||
|
#define DMA_DAR_LOW_OFF_RDCH 0x114
|
||||||
|
#define DMA_DAR_HIGH_OFF_RDCH 0x118
|
||||||
|
#define DMA_LLP_LOW_OFF_RDCH 0x11C
|
||||||
|
#define DMA_LLP_HIGH_OFF_RDCH 0x120
|
||||||
|
|
||||||
|
struct sanity_data {
|
||||||
|
u32 size;
|
||||||
|
u32 crc;
|
||||||
|
};
|
||||||
|
|
||||||
|
/* First 1MB of BAR0 is reserved for control data */
|
||||||
|
struct pcie_epf_bar0 {
|
||||||
|
/* RP system memory allocated for EP DMA operations */
|
||||||
|
u64 rp_phy_addr;
|
||||||
|
/* EP system memory allocated as BAR */
|
||||||
|
u64 ep_phy_addr;
|
||||||
|
/* MSI data for RP -> EP interrupts */
|
||||||
|
u32 msi_data[DMA_WR_CHNL_NUM + DMA_RD_CHNL_NUM];
|
||||||
|
struct sanity_data wr_data[DMA_WR_CHNL_NUM];
|
||||||
|
struct sanity_data rd_data[DMA_RD_CHNL_NUM];
|
||||||
|
};
|
||||||
|
|
||||||
|
struct dma_ll_ctrl {
|
||||||
|
u32 cb:1;
|
||||||
|
u32 tcb:1;
|
||||||
|
u32 llp:1;
|
||||||
|
u32 lie:1;
|
||||||
|
u32 rie:1;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct dma_ll {
|
||||||
|
volatile struct dma_ll_ctrl ele;
|
||||||
|
u32 size;
|
||||||
|
u32 src_low;
|
||||||
|
u32 src_high;
|
||||||
|
u32 dst_low;
|
||||||
|
u32 dst_high;
|
||||||
|
};
|
||||||
|
|
||||||
|
static inline void dma_common_wr16(void __iomem *p, u32 val, u32 offset)
|
||||||
|
{
|
||||||
|
writew(val, offset + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline u16 dma_common_rd16(void __iomem *p, u32 offset)
|
||||||
|
{
|
||||||
|
return readw(offset + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void dma_common_wr(void __iomem *p, u32 val, u32 offset)
|
||||||
|
{
|
||||||
|
writel(val, offset + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline u32 dma_common_rd(void __iomem *p, u32 offset)
|
||||||
|
{
|
||||||
|
return readl(offset + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void dma_channel_wr(void __iomem *p, u8 channel, u32 val,
|
||||||
|
u32 offset)
|
||||||
|
{
|
||||||
|
writel(val, (0x200 * (channel + 1)) + offset + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline u32 dma_channel_rd(void __iomem *p, u8 channel, u32 offset)
|
||||||
|
{
|
||||||
|
return readl((0x200 * (channel + 1)) + offset + p);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
291
include/linux/tegra-pcie-edma-test-common.h
Normal file
291
include/linux/tegra-pcie-edma-test-common.h
Normal file
@@ -0,0 +1,291 @@
|
|||||||
|
/* SPDX-License-Identifier: GPL-2.0+ */
|
||||||
|
/*
|
||||||
|
* PCIe DMA EPF Library for Tegra PCIe
|
||||||
|
*
|
||||||
|
* Copyright (C) 2022 NVIDIA Corporation. All rights reserved.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef TEGRA_PCIE_EDMA_TEST_COMMON_H
|
||||||
|
#define TEGRA_PCIE_EDMA_TEST_COMMON_H
|
||||||
|
|
||||||
|
#include <linux/pci-epf.h>
|
||||||
|
#include <linux/pcie_dma.h>
|
||||||
|
#include <linux/tegra-pcie-edma.h>
|
||||||
|
|
||||||
|
#define EDMA_ABORT_TEST_EN (edma->edma_ch & 0x40000000)
|
||||||
|
#define IS_EDMA_CH_ENABLED(i) (edma->edma_ch & ((BIT(i) << 4)))
|
||||||
|
#define IS_EDMA_CH_ASYNC(i) (edma->edma_ch & BIT(i))
|
||||||
|
#define REMOTE_EDMA_TEST_EN (edma->edma_ch & 0x80000000)
|
||||||
|
#define EDMA_PERF (edma->tsz / (diff / 1000))
|
||||||
|
#define EDMA_CPERF ((edma->tsz * (edma->nents / edma->nents_per_ch)) / (diff / 1000))
|
||||||
|
|
||||||
|
#define EDMA_PRIV_CH_OFF 16
|
||||||
|
#define EDMA_PRIV_LR_OFF 20
|
||||||
|
#define EDMA_PRIV_XF_OFF 21
|
||||||
|
|
||||||
|
struct edmalib_common {
|
||||||
|
struct device *fdev;
|
||||||
|
void *bar0_virt;
|
||||||
|
void *src_virt;
|
||||||
|
void __iomem *dma_base;
|
||||||
|
u32 dma_size;
|
||||||
|
dma_addr_t src_dma_addr;
|
||||||
|
dma_addr_t dst_dma_addr;
|
||||||
|
dma_addr_t bar0_phy;
|
||||||
|
u32 stress_count;
|
||||||
|
void *cookie;
|
||||||
|
struct device_node *of_node;
|
||||||
|
wait_queue_head_t wr_wq[DMA_WR_CHNL_NUM];
|
||||||
|
wait_queue_head_t rd_wq[DMA_RD_CHNL_NUM];
|
||||||
|
unsigned long wr_busy;
|
||||||
|
unsigned long rd_busy;
|
||||||
|
ktime_t edma_start_time[DMA_WR_CHNL_NUM];
|
||||||
|
u64 tsz;
|
||||||
|
u32 edma_ch;
|
||||||
|
u32 prev_edma_ch;
|
||||||
|
u32 nents;
|
||||||
|
struct tegra_pcie_edma_desc *ll_desc;
|
||||||
|
int priv_iter[DMA_WR_CHNL_NUM];
|
||||||
|
struct pcie_tegra_edma_remote_info edma_remote;
|
||||||
|
u32 nents_per_ch;
|
||||||
|
u32 st_as_ch;
|
||||||
|
u32 ls_as_ch;
|
||||||
|
};
|
||||||
|
|
||||||
|
static struct edmalib_common *l_edma;
|
||||||
|
|
||||||
|
static void edma_final_complete(void *priv, edma_xfer_status_t status,
|
||||||
|
struct tegra_pcie_edma_desc *desc)
|
||||||
|
{
|
||||||
|
struct edmalib_common *edma = l_edma;
|
||||||
|
int cb = *(int *)priv;
|
||||||
|
u32 ch = (cb >> EDMA_PRIV_CH_OFF) & 0xF;
|
||||||
|
edma_xfer_type_t xfer_type = (cb >> EDMA_PRIV_XF_OFF) & 0x1;
|
||||||
|
char *xfer_str[2] = {"WR", "RD"};
|
||||||
|
u32 l_r = (cb >> EDMA_PRIV_LR_OFF) & 0x1;
|
||||||
|
char *l_r_str[2] = {"local", "remote"};
|
||||||
|
u64 diff = ktime_to_ns(ktime_get()) - ktime_to_ns(edma->edma_start_time[ch]);
|
||||||
|
u64 cdiff = ktime_to_ns(ktime_get()) - ktime_to_ns(edma->edma_start_time[edma->st_as_ch]);
|
||||||
|
|
||||||
|
cb = cb & 0xFFFF;
|
||||||
|
if (EDMA_ABORT_TEST_EN && status == EDMA_XFER_SUCCESS)
|
||||||
|
dma_common_wr(edma->dma_base, DMA_WRITE_DOORBELL_OFF_WR_STOP | (ch + 1),
|
||||||
|
DMA_WRITE_DOORBELL_OFF);
|
||||||
|
|
||||||
|
dev_info(edma->fdev, "%s: %s-%s-Async complete for chan %d with status %d. Total desc %d of Sz %d Bytes done in time %llu nsec. Perf is %llu Mbps\n",
|
||||||
|
__func__, xfer_str[xfer_type], l_r_str[l_r], ch, status, edma->nents_per_ch*(cb+1),
|
||||||
|
edma->dma_size, diff, EDMA_PERF);
|
||||||
|
|
||||||
|
if (ch == edma->ls_as_ch)
|
||||||
|
dev_info(edma->fdev, "%s: All Async channels. Cumulative Perf %llu Mbps, time %llu nsec\n",
|
||||||
|
__func__, EDMA_CPERF, cdiff);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void edma_complete(void *priv, edma_xfer_status_t status, struct tegra_pcie_edma_desc *desc)
|
||||||
|
{
|
||||||
|
struct edmalib_common *edma = l_edma;
|
||||||
|
int cb = *(int *)priv;
|
||||||
|
u32 ch = (cb >> 16);
|
||||||
|
|
||||||
|
if (BIT(ch) & edma->wr_busy) {
|
||||||
|
edma->wr_busy &= ~(BIT(ch));
|
||||||
|
wake_up(&edma->wr_wq[ch]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status == 0)
|
||||||
|
dev_dbg(edma->fdev, "%s: status %d, cb %d\n", __func__, status, cb);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* debugfs to perform eDMA lib transfers and do CRC check */
|
||||||
|
static int edmalib_common_test(struct edmalib_common *edma)
|
||||||
|
{
|
||||||
|
struct tegra_pcie_edma_desc *ll_desc = edma->ll_desc;
|
||||||
|
dma_addr_t src_dma_addr = edma->src_dma_addr;
|
||||||
|
dma_addr_t dst_dma_addr = edma->dst_dma_addr;
|
||||||
|
u32 nents = edma->nents, num_chans = 0, nents_per_ch = 0, nent_id = 0, chan_count;
|
||||||
|
u32 i, j, k, max_size, db_off, num_descriptors;
|
||||||
|
edma_xfer_status_t ret;
|
||||||
|
struct tegra_pcie_edma_init_info info = {};
|
||||||
|
struct tegra_pcie_edma_chans_info *chan_info;
|
||||||
|
struct tegra_pcie_edma_xfer_info tx_info = {};
|
||||||
|
u64 diff;
|
||||||
|
edma_xfer_type_t xfer_type;
|
||||||
|
char *xfer_str[2] = {"WR", "RD"};
|
||||||
|
u32 l_r;
|
||||||
|
char *l_r_str[2] = {"local", "remote"};
|
||||||
|
|
||||||
|
if (!edma->stress_count) {
|
||||||
|
tegra_pcie_edma_deinit(edma->cookie);
|
||||||
|
edma->cookie = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
l_edma = edma;
|
||||||
|
|
||||||
|
if (EDMA_ABORT_TEST_EN) {
|
||||||
|
edma->edma_ch &= ~0xFF;
|
||||||
|
/* only channel 0, 2 is ASYNC, where chan 0 async gets aborted */
|
||||||
|
edma->edma_ch |= 0xF5;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (edma->cookie && edma->prev_edma_ch != edma->edma_ch) {
|
||||||
|
edma->st_as_ch = -1;
|
||||||
|
dev_info(edma->fdev, "edma_ch changed from 0x%x != 0x%x, deinit\n",
|
||||||
|
edma->prev_edma_ch, edma->edma_ch);
|
||||||
|
tegra_pcie_edma_deinit(edma->cookie);
|
||||||
|
edma->cookie = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
info.np = edma->of_node;
|
||||||
|
|
||||||
|
if (REMOTE_EDMA_TEST_EN) {
|
||||||
|
num_descriptors = 1024;
|
||||||
|
info.rx[0].desc_phy_base = edma->bar0_phy + SZ_512K;
|
||||||
|
info.rx[0].desc_iova = 0xf0000000 + SZ_512K;
|
||||||
|
info.rx[1].desc_phy_base = edma->bar0_phy + SZ_512K + SZ_256K;
|
||||||
|
info.rx[1].desc_iova = 0xf0000000 + SZ_512K + SZ_256K;
|
||||||
|
info.edma_remote = &edma->edma_remote;
|
||||||
|
chan_count = DMA_RD_CHNL_NUM;
|
||||||
|
chan_info = &info.rx[0];
|
||||||
|
xfer_type = EDMA_XFER_READ;
|
||||||
|
db_off = DMA_WRITE_DOORBELL_OFF;
|
||||||
|
l_r = 1;
|
||||||
|
} else {
|
||||||
|
chan_count = DMA_WR_CHNL_NUM;
|
||||||
|
num_descriptors = 4096;
|
||||||
|
chan_info = &info.tx[0];
|
||||||
|
xfer_type = EDMA_XFER_WRITE;
|
||||||
|
db_off = DMA_READ_DOORBELL_OFF;
|
||||||
|
l_r = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (i = 0; i < chan_count; i++) {
|
||||||
|
struct tegra_pcie_edma_chans_info *ch = chan_info + i;
|
||||||
|
|
||||||
|
ch->ch_type = IS_EDMA_CH_ASYNC(i) ? EDMA_CHAN_XFER_ASYNC :
|
||||||
|
EDMA_CHAN_XFER_SYNC;
|
||||||
|
if (IS_EDMA_CH_ENABLED(i)) {
|
||||||
|
if (edma->st_as_ch == -1)
|
||||||
|
edma->st_as_ch = i;
|
||||||
|
edma->ls_as_ch = i;
|
||||||
|
ch->num_descriptors = num_descriptors;
|
||||||
|
num_chans++;
|
||||||
|
} else
|
||||||
|
ch->num_descriptors = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
max_size = (BAR0_DMA_BUF_SIZE - BAR0_DMA_BUF_OFFSET) / 2;
|
||||||
|
if (((edma->dma_size * nents) > max_size) || (nents > NUM_EDMA_DESC)) {
|
||||||
|
dev_err(edma->fdev, "%s: max dma size including all nents(%d), max_nents(%d), dma_size(%d) should be <= 0x%x\n",
|
||||||
|
__func__, nents, NUM_EDMA_DESC, edma->dma_size, max_size);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
nents_per_ch = nents / num_chans;
|
||||||
|
if (nents_per_ch == 0) {
|
||||||
|
dev_err(edma->fdev, "%s: nents(%d) < enabled chanes(%d)\n",
|
||||||
|
__func__, nents, num_chans);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (j = 0; j < nents; j++) {
|
||||||
|
ll_desc->src = src_dma_addr + (j * edma->dma_size);
|
||||||
|
ll_desc->dst = dst_dma_addr + (j * edma->dma_size);
|
||||||
|
dev_dbg(edma->fdev, "src %llx, dst %llx at %d\n", ll_desc->src, ll_desc->dst, j);
|
||||||
|
ll_desc->sz = edma->dma_size;
|
||||||
|
ll_desc++;
|
||||||
|
}
|
||||||
|
ll_desc = edma->ll_desc;
|
||||||
|
|
||||||
|
edma->tsz = (u64)edma->stress_count * (nents_per_ch) * (u64)edma->dma_size * 8UL;
|
||||||
|
|
||||||
|
if (!edma->cookie && (edma->prev_edma_ch != edma->edma_ch)) {
|
||||||
|
dev_info(edma->fdev, "%s: re-init edma lib prev_ch(%x) != current chans(%x)\n",
|
||||||
|
__func__, edma->prev_edma_ch, edma->edma_ch);
|
||||||
|
edma->cookie = tegra_pcie_edma_initialize(&info);
|
||||||
|
edma->prev_edma_ch = edma->edma_ch;
|
||||||
|
}
|
||||||
|
|
||||||
|
edma->nents_per_ch = nents_per_ch;
|
||||||
|
|
||||||
|
/* generate random bytes to transfer */
|
||||||
|
get_random_bytes(edma->src_virt, edma->dma_size * nents_per_ch);
|
||||||
|
dev_info(edma->fdev, "%s: EDMA LIB %s started for %d chans, size %d Bytes, iterations: %d of descriptors %d\n",
|
||||||
|
__func__, xfer_str[xfer_type], num_chans, edma->dma_size, edma->stress_count,
|
||||||
|
nents_per_ch);
|
||||||
|
|
||||||
|
/* LL DMA with size epfnv->dma_size per desc */
|
||||||
|
for (i = 0; i < chan_count; i++) {
|
||||||
|
int ch = i;
|
||||||
|
struct tegra_pcie_edma_chans_info *ch_info = chan_info + i;
|
||||||
|
|
||||||
|
if (ch_info->num_descriptors == 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
edma->edma_start_time[i] = ktime_get();
|
||||||
|
tx_info.desc = &ll_desc[nent_id++ * nents_per_ch];
|
||||||
|
for (k = 0; k < edma->stress_count; k++) {
|
||||||
|
tx_info.channel_num = ch;
|
||||||
|
tx_info.type = xfer_type;
|
||||||
|
tx_info.nents = nents_per_ch;
|
||||||
|
if (ch_info->ch_type == EDMA_CHAN_XFER_ASYNC) {
|
||||||
|
if (k == edma->stress_count - 1)
|
||||||
|
tx_info.complete = edma_final_complete;
|
||||||
|
else
|
||||||
|
tx_info.complete = edma_complete;
|
||||||
|
}
|
||||||
|
edma->priv_iter[ch] = k | (xfer_type << EDMA_PRIV_XF_OFF) |
|
||||||
|
(l_r << EDMA_PRIV_LR_OFF) | (ch << EDMA_PRIV_CH_OFF);
|
||||||
|
tx_info.priv = &edma->priv_iter[ch];
|
||||||
|
ret = tegra_pcie_edma_submit_xfer(edma->cookie, &tx_info);
|
||||||
|
if (ret == EDMA_XFER_FAIL_NOMEM) {
|
||||||
|
/** Retry after 20 msec */
|
||||||
|
dev_dbg(edma->fdev, "%s: EDMA_XFER_FAIL_NOMEM stress count %d on channel %d iter %d\n",
|
||||||
|
__func__, edma->stress_count, i, k);
|
||||||
|
ret = wait_event_timeout(edma->wr_wq[i],
|
||||||
|
!(edma->wr_busy & (1 << i)),
|
||||||
|
msecs_to_jiffies(500));
|
||||||
|
/* Do a more sleep to avoid repeated wait and wake calls */
|
||||||
|
msleep(100);
|
||||||
|
if (ret == 0) {
|
||||||
|
dev_err(edma->fdev, "%s: %d timedout\n", __func__, i);
|
||||||
|
ret = -ETIMEDOUT;
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
k--;
|
||||||
|
continue;
|
||||||
|
} else if ((ret != EDMA_XFER_SUCCESS) && (ret != EDMA_XFER_FAIL_NOMEM)) {
|
||||||
|
dev_err(edma->fdev, "%s: LL %d, SZ: %u B CH: %d failed. %d at iter %d ret: %d\n",
|
||||||
|
__func__, xfer_type, edma->dma_size, ch, ret, k, ret);
|
||||||
|
if (EDMA_ABORT_TEST_EN) {
|
||||||
|
msleep(5000);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
dev_dbg(edma->fdev, "%s: LL EDMA LIB %d, SZ: %u B CH: %d iter %d\n",
|
||||||
|
__func__, xfer_type, edma->dma_size, ch, i);
|
||||||
|
}
|
||||||
|
if (EDMA_ABORT_TEST_EN && i == 0) {
|
||||||
|
msleep(edma->stress_count);
|
||||||
|
dma_common_wr(edma->dma_base, DMA_WRITE_DOORBELL_OFF_WR_STOP,
|
||||||
|
db_off);
|
||||||
|
}
|
||||||
|
diff = ktime_to_ns(ktime_get()) - ktime_to_ns(edma->edma_start_time[i]);
|
||||||
|
if (ch_info->ch_type == EDMA_CHAN_XFER_SYNC)
|
||||||
|
dev_info(edma->fdev, "%s: EDMA LIB %s-%s-SYNC done for %d iter on channel %d. Total Size %llu bytes, time %llu nsec. Perf is %llu Mbps\n",
|
||||||
|
__func__, xfer_str[xfer_type], l_r_str[l_r], edma->stress_count, i,
|
||||||
|
edma->tsz, diff, EDMA_PERF);
|
||||||
|
}
|
||||||
|
dev_info(edma->fdev, "%s: EDMA LIB submit done\n", __func__);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
fail:
|
||||||
|
if (ret != EDMA_XFER_DEINIT) {
|
||||||
|
tegra_pcie_edma_deinit(edma->cookie);
|
||||||
|
edma->cookie = NULL;
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* TEGRA_PCIE_EDMA_TEST_COMMON_H */
|
||||||
Reference in New Issue
Block a user