nv-tegra-mirror/linux-nv-oot
1
0
Fork 0
mirror of git://nv-tegra.nvidia.com/linux-nv-oot.git synced 2025-12-22 17:25:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

nvethernet: Create pool per Rx DMA channel

Browse Source 
Existing implementation uses a single page_pool
for all Rx DMA channels. As by default all irqs
are on CPU 0, this does not cause any issue.
Routing an irq to some other CPU causes race
conditon for page allocation from page_pool which
leads to random memory corruption.

[  158.416637] Call trace:
[  158.416644]  arm_lpae_map_pages+0xb4/0x1e0
[  158.416649]  arm_smmu_map_pages+0x8c/0x160
[  158.416661]  __iommu_map+0xf8/0x2b0
[  158.416677]  iommu_map_atomic+0x58/0xb0
[  158.416683]  __iommu_dma_map+0xac/0x150
[  158.416687]  iommu_dma_map_page+0xf4/0x220
[  158.416690]  dma_map_page_attrs+0x1e8/0x260
[  158.416727]  page_pool_dma_map+0x48/0xd0
[  158.416750]  __page_pool_alloc_pages_slow+0xc4/0x390
[  158.416757]  page_pool_alloc_pages+0x64/0x90
[  158.416762]  ether_padctrl_mii_rx_pins+0x164/0x9d0 [nvethernet]
[  158.416807]  ether_padctrl_mii_rx_pins+0x478/0x9d0 [nvethernet]
[  158.416822]  osi_process_rx_completions+0x284/0x4d0 [nvethernet]
[  158.416832]  0xffffa26218b8f71c
[  158.416855]  __napi_poll+0x48/0x230
[  158.416871]  net_rx_action+0xf4/0x290
[  158.416875]  __do_softirq+0x130/0x3e8
[  158.416889]  __irq_exit_rcu+0xe8/0x110

This change creates a page_pool per Rx DMA channel.
This ensures there is no race conditon for page
alloc/dealloc from each DMA napi context.

Bug 4541158

Change-Id: I12668ee7d824fd30d54a874bbbdf190d02943478
Signed-off-by: Aniruddha Paul <anpaul@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/linux-nv-oot/+/3093534
GVS: Gerrit_Virtual_Submit <buildbot_gerritrpt@nvidia.com>
Reviewed-by: Jon Hunter <jonathanh@nvidia.com>
Tested-by: Jon Hunter <jonathanh@nvidia.com>
This commit is contained in:
Aniruddha Paul
2024-03-07 12:35:37 +00:00
committed by mobile promotions
parent 003bd8ed9b
commit d01b6a3a2b
3 changed files with 56 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
drivers/net/ethernet/nvidia/nvethernet/ether_linux.c
View File

@@ -1651,7 +1651,7 @@ static int ether_request_irqs(struct ether_priv_data *pdata)
netdev_name(pdata->ndev), i); netdev_name(pdata->ndev), i);
ret = devm_request_irq(pdata->dev, pdata->vm_irqs[i], ret = devm_request_irq(pdata->dev, pdata->vm_irqs[i],
ether_vm_isr, ether_vm_isr,
IRQF_TRIGGER_NONE | IRQF_NOBALANCING, IRQF_TRIGGER_NONE,
pdata->irq_names[j++], pdata->irq_names[j++],
&pdata->vm_irq_data[i]); &pdata->vm_irq_data[i]);
if (unlikely(ret < 0)) { if (unlikely(ret < 0)) {
@@ -1770,7 +1770,8 @@ static void ether_napi_enable(struct ether_priv_data *pdata)
static void ether_free_rx_skbs(struct osi_rx_swcx *rx_swcx, static void ether_free_rx_skbs(struct osi_rx_swcx *rx_swcx,
struct ether_priv_data *pdata, struct ether_priv_data *pdata,
unsigned int rx_buf_len, unsigned int rx_buf_len,
void *resv_buf_virt_addr) void *resv_buf_virt_addr,
unsigned int chan)
{ {
struct osi_dma_priv_data *osi_dma = pdata->osi_dma; struct osi_dma_priv_data *osi_dma = pdata->osi_dma;
struct osi_rx_swcx *prx_swcx = NULL; struct osi_rx_swcx *prx_swcx = NULL;
@@ -1782,9 +1783,10 @@ static void ether_free_rx_skbs(struct osi_rx_swcx *rx_swcx,
if (prx_swcx->buf_virt_addr != NULL) { if (prx_swcx->buf_virt_addr != NULL) {
if (resv_buf_virt_addr != prx_swcx->buf_virt_addr) { if (resv_buf_virt_addr != prx_swcx->buf_virt_addr) {
#ifdef ETHER_PAGE_POOL #ifdef ETHER_PAGE_POOL
page_pool_put_full_page(pdata->page_pool, if (chan != ETHER_INVALID_CHAN_NUM)
prx_swcx->buf_virt_addr, page_pool_put_full_page(pdata->page_pool[chan],
false); prx_swcx->buf_virt_addr,
false);
#else #else
dma_unmap_single(pdata->dev, dma_unmap_single(pdata->dev,
prx_swcx->buf_phy_addr, prx_swcx->buf_phy_addr,
@@ -1811,16 +1813,18 @@ static void free_rx_dma_resources(struct osi_dma_priv_data *osi_dma,
{ {
unsigned long rx_desc_size = sizeof(struct osi_rx_desc) * osi_dma->rx_ring_sz; unsigned long rx_desc_size = sizeof(struct osi_rx_desc) * osi_dma->rx_ring_sz;
struct osi_rx_ring *rx_ring = NULL; struct osi_rx_ring *rx_ring = NULL;
unsigned int i; unsigned int i, chan;
for (i = 0; i < OSI_MGBE_MAX_NUM_CHANS; i++) { for (i = 0; i < OSI_MGBE_MAX_NUM_CHANS; i++) {
rx_ring = osi_dma->rx_ring[i]; rx_ring = osi_dma->rx_ring[i];
chan = osi_dma->dma_chans[i];
if (rx_ring != NULL) { if (rx_ring != NULL) {
if (rx_ring->rx_swcx != NULL) { if (rx_ring->rx_swcx != NULL) {
ether_free_rx_skbs(rx_ring->rx_swcx, pdata, ether_free_rx_skbs(rx_ring->rx_swcx, pdata,
osi_dma->rx_buf_len, osi_dma->rx_buf_len,
pdata->resv_buf_virt_addr); pdata->resv_buf_virt_addr,
chan);
kfree(rx_ring->rx_swcx); kfree(rx_ring->rx_swcx);
} }
@@ -1832,13 +1836,13 @@ static void free_rx_dma_resources(struct osi_dma_priv_data *osi_dma,
kfree(rx_ring); kfree(rx_ring);
osi_dma->rx_ring[i] = NULL; osi_dma->rx_ring[i] = NULL;
} }
}
#ifdef ETHER_PAGE_POOL #ifdef ETHER_PAGE_POOL
if (pdata->page_pool) { if (chan != ETHER_INVALID_CHAN_NUM && pdata->page_pool[chan]) {
page_pool_destroy(pdata->page_pool); page_pool_destroy(pdata->page_pool[chan]);
pdata->page_pool = NULL; pdata->page_pool[chan] = NULL;
} }
#endif #endif
}
} }
/** /**
@@ -1915,7 +1919,8 @@ err_rx_desc:
* @retval "negative value" on failure. * @retval "negative value" on failure.
*/ */
static int ether_allocate_rx_buffers(struct ether_priv_data *pdata, static int ether_allocate_rx_buffers(struct ether_priv_data *pdata,
struct osi_rx_ring *rx_ring) struct osi_rx_ring *rx_ring,
unsigned int chan)
{ {
#ifndef ETHER_PAGE_POOL #ifndef ETHER_PAGE_POOL
unsigned int rx_buf_len = pdata->osi_dma->rx_buf_len; unsigned int rx_buf_len = pdata->osi_dma->rx_buf_len;
@@ -1934,15 +1939,17 @@ static int ether_allocate_rx_buffers(struct ether_priv_data *pdata,
rx_swcx = rx_ring->rx_swcx + i; rx_swcx = rx_ring->rx_swcx + i;
#ifdef ETHER_PAGE_POOL #ifdef ETHER_PAGE_POOL
page = page_pool_dev_alloc_pages(pdata->page_pool); if (chan != ETHER_INVALID_CHAN_NUM) {
if (!page) { page = page_pool_dev_alloc_pages(pdata->page_pool[chan]);
dev_err(pdata->dev, if (!page) {
"failed to allocate page pool buffer"); dev_err(pdata->dev,
return -ENOMEM; "failed to allocate page pool buffer");
} return -ENOMEM;
}
dma_addr = page_pool_get_dma_addr(page); dma_addr = page_pool_get_dma_addr(page);
rx_swcx->buf_virt_addr = page; rx_swcx->buf_virt_addr = page;
}
#else #else
skb = __netdev_alloc_skb_ip_align(pdata->ndev, rx_buf_len, skb = __netdev_alloc_skb_ip_align(pdata->ndev, rx_buf_len,
GFP_KERNEL); GFP_KERNEL);
@@ -1969,34 +1976,36 @@ static int ether_allocate_rx_buffers(struct ether_priv_data *pdata,
#ifdef ETHER_PAGE_POOL #ifdef ETHER_PAGE_POOL
/** /**
* @brief Create Rx buffer page pool * @brief Create Rx buffer page pool per channel
* *
* Algorithm: Invokes page pool API to create Rx buffer pool. * Algorithm: Invokes page pool API to create Rx buffer pool.
* *
* @param[in] pdata: OSD private data. * @param[in] pdata: OSD private data.
* @param[chan] chan: Rx DMA channel number.
* *
* @retval 0 on success * @retval 0 on success
* @retval "negative value" on failure. * @retval "negative value" on failure.
*/ */
static int ether_page_pool_create(struct ether_priv_data *pdata) static int ether_page_pool_create_per_chan(struct ether_priv_data *pdata,
unsigned int chan)
{ {
struct osi_dma_priv_data *osi_dma = pdata->osi_dma; struct osi_dma_priv_data *osi_dma = pdata->osi_dma;
struct page_pool_params pp_params = { 0 }; struct page_pool_params pp_params = { 0 };
unsigned int num_pages; unsigned int num_pages, pool_size = 1024;
int ret = 0; int ret = 0;
pp_params.flags = PP_FLAG_DMA_MAP; pp_params.flags = PP_FLAG_DMA_MAP;
pp_params.pool_size = osi_dma->rx_buf_len; pp_params.pool_size = pool_size;
num_pages = DIV_ROUND_UP(osi_dma->rx_buf_len, PAGE_SIZE); num_pages = DIV_ROUND_UP(osi_dma->rx_buf_len, PAGE_SIZE);
pp_params.order = ilog2(roundup_pow_of_two(num_pages)); pp_params.order = ilog2(roundup_pow_of_two(num_pages));
pp_params.nid = dev_to_node(pdata->dev); pp_params.nid = dev_to_node(pdata->dev);
pp_params.dev = pdata->dev; pp_params.dev = pdata->dev;
pp_params.dma_dir = DMA_FROM_DEVICE; pp_params.dma_dir = DMA_FROM_DEVICE;
pdata->page_pool = page_pool_create(&pp_params); pdata->page_pool[chan] = page_pool_create(&pp_params);
if (IS_ERR(pdata->page_pool)) { if (IS_ERR(pdata->page_pool[chan])) {
ret = PTR_ERR(pdata->page_pool); ret = PTR_ERR(pdata->page_pool[chan]);
pdata->page_pool = NULL; pdata->page_pool[chan] = NULL;
return ret; return ret;
} }
@@ -2025,18 +2034,19 @@ static int ether_allocate_rx_dma_resources(struct osi_dma_priv_data *osi_dma,
unsigned int i; unsigned int i;
int ret = 0; int ret = 0;
#ifdef ETHER_PAGE_POOL
ret = ether_page_pool_create(pdata);
if (ret < 0) {
pr_err("%s(): failed to create page pool\n",
__func__);
goto exit;
}
#endif
for (i = 0; i < OSI_MGBE_MAX_NUM_CHANS; i++) { for (i = 0; i < OSI_MGBE_MAX_NUM_CHANS; i++) {
chan = osi_dma->dma_chans[i]; chan = osi_dma->dma_chans[i];
if (chan != ETHER_INVALID_CHAN_NUM) { if (chan != ETHER_INVALID_CHAN_NUM) {
#ifdef ETHER_PAGE_POOL
ret = ether_page_pool_create_per_chan(pdata, chan);
if (ret < 0) {
pr_err("%s(): failed to create page pool\n",
__func__);
goto exit;
}
#endif
ret = allocate_rx_dma_resource(osi_dma, pdata->dev, ret = allocate_rx_dma_resource(osi_dma, pdata->dev,
chan); chan);
if (ret != 0) { if (ret != 0) {
@@ -2044,7 +2054,8 @@ static int ether_allocate_rx_dma_resources(struct osi_dma_priv_data *osi_dma,
} }
ret = ether_allocate_rx_buffers(pdata, ret = ether_allocate_rx_buffers(pdata,
osi_dma->rx_ring[chan]); osi_dma->rx_ring[chan],
chan);
if (ret < 0) { if (ret < 0) {
goto exit; goto exit;
} }

4
drivers/net/ethernet/nvidia/nvethernet/ether_linux.h
View File

@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: GPL-2.0-only */ /* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (c) 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved */ /* Copyright (c) 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved */
#ifndef ETHER_LINUX_H #ifndef ETHER_LINUX_H
#define ETHER_LINUX_H #define ETHER_LINUX_H
@@ -576,7 +576,7 @@ struct ether_priv_data {
struct ether_vm_irq_data *vm_irq_data; struct ether_vm_irq_data *vm_irq_data;
#ifdef ETHER_PAGE_POOL #ifdef ETHER_PAGE_POOL
/** Pointer to page pool */ /** Pointer to page pool */
struct page_pool *page_pool; struct page_pool *page_pool[OSI_MGBE_MAX_NUM_CHANS];
#endif #endif
#ifdef CONFIG_DEBUG_FS #ifdef CONFIG_DEBUG_FS
/** Debug fs directory pointer */ /** Debug fs directory pointer */

10
drivers/net/ethernet/nvidia/nvethernet/osd.c
View File

@@ -1,5 +1,5 @@
// SPDX-License-Identifier: GPL-2.0-only // SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved */ /* Copyright (c) 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved */
#include "ether_linux.h" #include "ether_linux.h"
@@ -240,7 +240,7 @@ static inline int ether_alloc_skb(struct ether_priv_data *pdata,
} }
#else #else
rx_swcx->buf_virt_addr = page_pool_dev_alloc_pages(pdata->page_pool); rx_swcx->buf_virt_addr = page_pool_dev_alloc_pages(pdata->page_pool[chan]);
if (!rx_swcx->buf_virt_addr) { if (!rx_swcx->buf_virt_addr) {
dev_err(pdata->dev, dev_err(pdata->dev,
"page pool allocation failed using resv_buf\n"); "page pool allocation failed using resv_buf\n");
@@ -654,7 +654,7 @@ static void osd_receive_packet(void *priv, struct osi_rx_ring *rx_ring,
dev_err(pdata->dev, dev_err(pdata->dev,
"%s(): Error in allocating the skb\n", "%s(): Error in allocating the skb\n",
__func__); __func__);
page_pool_recycle_direct(pdata->page_pool, page); page_pool_recycle_direct(pdata->page_pool[chan], page);
return; return;
} }
@@ -663,7 +663,7 @@ static void osd_receive_packet(void *priv, struct osi_rx_ring *rx_ring,
skb_copy_to_linear_data(skb, page_address(page), skb_copy_to_linear_data(skb, page_address(page),
rx_pkt_cx->pkt_len); rx_pkt_cx->pkt_len);
skb_put(skb, rx_pkt_cx->pkt_len); skb_put(skb, rx_pkt_cx->pkt_len);
page_pool_recycle_direct(pdata->page_pool, page); page_pool_recycle_direct(pdata->page_pool[chan], page);
#else #else
skb_put(skb, rx_pkt_cx->pkt_len); skb_put(skb, rx_pkt_cx->pkt_len);
#endif #endif
@@ -717,7 +717,7 @@ static void osd_receive_packet(void *priv, struct osi_rx_ring *rx_ring,
ndev->stats.rx_fifo_errors = osi_core->mmc.mmc_rx_fifo_overflow; ndev->stats.rx_fifo_errors = osi_core->mmc.mmc_rx_fifo_overflow;
ndev->stats.rx_errors++; ndev->stats.rx_errors++;
#ifdef ETHER_PAGE_POOL #ifdef ETHER_PAGE_POOL
page_pool_recycle_direct(pdata->page_pool, page); page_pool_recycle_direct(pdata->page_pool[chan], page);
#endif #endif
dev_kfree_skb_any(skb); dev_kfree_skb_any(skb);
} }