From 41429e6df9da0cfb8a2be3854822a8722adbd230 Mon Sep 17 00:00:00 2001 From: Bhadram Varka Date: Thu, 27 May 2021 19:28:35 +0530 Subject: [PATCH] nvethernet: Support of NVGRO for NVUSO segments Algorithm: - There is one master queue and one final queue to maintain the packets. - Final queue will start when a packet received with TTL=1 - TTL = 2 and 3 packets will be queued into final if those are in sequence else those will be queued to master queue. - When packet received with TTL = 2 then only merge process will start - For every valid final queue merge starts by looking at the IPID in master queue in increment order. This loop continues until TTL = 2 found in master queue. - Once TTL = 2 found then entire final queue will be given GRO layer in sequential order. Bug 3106903 Change-Id: I6cc08f3faae67a04813d32d59f7cce02f06dccbf Signed-off-by: Bhadram Varka Reviewed-on: https://git-master.nvidia.com/r/c/linux-nvidia/+/2554423 Tested-by: mobile promotions Reviewed-by: svcacv Reviewed-by: Nagarjuna Kristam Reviewed-by: Narayan Reddy Reviewed-by: Sachin Nikam Reviewed-by: mobile promotions --- .../ethernet/nvidia/nvethernet/ether_linux.c | 19 ++ .../ethernet/nvidia/nvethernet/ether_linux.h | 37 +++ drivers/net/ethernet/nvidia/nvethernet/osd.c | 274 ++++++++++++++++++ .../net/ethernet/nvidia/nvethernet/sysfs.c | 179 ++++++++++++ 4 files changed, 509 insertions(+) diff --git a/drivers/net/ethernet/nvidia/nvethernet/ether_linux.c b/drivers/net/ethernet/nvidia/nvethernet/ether_linux.c index 37f08370..a82798f3 100644 --- a/drivers/net/ethernet/nvidia/nvethernet/ether_linux.c +++ b/drivers/net/ethernet/nvidia/nvethernet/ether_linux.c @@ -2423,6 +2423,11 @@ static int ether_open(struct net_device *dev) /* call function to schedule workqueue */ ether_stats_work_queue_start(pdata); +#ifdef ETHER_NVGRO + /* start NVGRO timer for purging */ + mod_timer(&pdata->nvgro_timer, + jiffies + msecs_to_jiffies(pdata->nvgro_timer_intrvl)); +#endif return ret; err_r_irq: @@ -2603,6 +2608,11 @@ static int ether_close(struct net_device *ndev) unsigned int chan = 0x0; int i; +#ifdef ETHER_NVGRO + del_timer_sync(&pdata->nvgro_timer); + /* TODO: purge the queues */ +#endif + /* Unregister broadcasting MAC timestamp to clients */ tegra_unregister_hwtime_source(); @@ -6091,6 +6101,15 @@ static int ether_probe(struct platform_device *pdev) INIT_LIST_HEAD(&pdata->tx_ts_skb_head); INIT_WORK(&pdata->tx_ts_work, ether_get_tx_ts); +#ifdef ETHER_NVGRO + __skb_queue_head_init(&pdata->mq); + __skb_queue_head_init(&pdata->fq); + pdata->pkt_age_msec = NVGRO_AGE_THRESHOLD; + pdata->nvgro_timer_intrvl = NVGRO_PURGE_TIMER_THRESHOLD; + pdata->nvgro_dropped = 0; + timer_setup(&pdata->nvgro_timer, ether_nvgro_purge_timer, 0); +#endif + return 0; err_sysfs: diff --git a/drivers/net/ethernet/nvidia/nvethernet/ether_linux.h b/drivers/net/ethernet/nvidia/nvethernet/ether_linux.h index 126413b3..a7fe4f39 100644 --- a/drivers/net/ethernet/nvidia/nvethernet/ether_linux.h +++ b/drivers/net/ethernet/nvidia/nvethernet/ether_linux.h @@ -65,6 +65,12 @@ #ifdef MACSEC_SUPPORT #include "macsec.h" #endif +#ifdef ETHER_NVGRO +#include +#include +#include +#endif /* ETHER_NVGRO */ + /** * @brief Max number of Ethernet IRQs supported in HW */ @@ -201,6 +207,14 @@ #define ETHER_ADDRESS_MAC 1 #define ETHER_ADDRESS_BC 0 +#ifdef ETHER_NVGRO +/* NVGRO packets purge threshold in msec */ +#define NVGRO_AGE_THRESHOLD 500 +#define NVGRO_PURGE_TIMER_THRESHOLD 5000 +#define NVGRO_RX_RUNNING OSI_BIT(0) +#define NVGRO_PURGE_TIMER_RUNNING OSI_BIT(1) +#endif + /** * @brief Check if Tx data buffer length is within bounds. * @@ -525,6 +539,26 @@ struct ether_priv_data { int phy_reset_post_delay; /** PHY reset duration delay */ int phy_reset_duration; +#ifdef ETHER_NVGRO + /** Master queue */ + struct sk_buff_head mq; + /** Master queue */ + struct sk_buff_head fq; + /** expected IP ID */ + u16 expected_ip_id; + /** Timer for purginging the packets in FQ and MQ based on threshold */ + struct timer_list nvgro_timer; + /** Rx processing state for NVGRO */ + atomic_t rx_state; + /** Purge timer state for NVGRO */ + atomic_t timer_state; + /** NVGRO packet age threshold in milseconds */ + u32 pkt_age_msec; + /** NVGRO purge timer interval */ + u32 nvgro_timer_intrvl; + /** NVGRO packet dropped count */ + u64 nvgro_dropped; +#endif }; /** @@ -681,4 +715,7 @@ void ether_set_rx_mode(struct net_device *dev); int ether_tc_setup_taprio(struct ether_priv_data *pdata, struct tc_taprio_qopt_offload *qopt); #endif +#ifdef ETHER_NVGRO +void ether_nvgro_purge_timer(struct timer_list *t); +#endif /* ETHER_NVGRO */ #endif /* ETHER_LINUX_H */ diff --git a/drivers/net/ethernet/nvidia/nvethernet/osd.c b/drivers/net/ethernet/nvidia/nvethernet/osd.c index 04330710..8c197816 100644 --- a/drivers/net/ethernet/nvidia/nvethernet/osd.c +++ b/drivers/net/ethernet/nvidia/nvethernet/osd.c @@ -253,6 +253,272 @@ static void osd_realloc_buf(void *priv, struct osi_rx_ring *rx_ring, ether_realloc_rx_skb(pdata, rx_ring, chan); } +#ifdef ETHER_NVGRO +/** + * @brief ether_gro_merge_complete - Merging the packets with GRO layer + * + * @param[in] nvgro_q: NVGRO packet sequence queue. + * @param[in] napi: Driver NAPI instance. + */ +static inline void ether_gro_merge_complete(struct sk_buff_head *nvgro_q, + struct napi_struct *napi) +{ + struct list_head h; + struct sk_buff *f_skb, *p, *pp; + + f_skb = __skb_peek(nvgro_q); + + INIT_LIST_HEAD(&h); + + skb_queue_walk_safe(nvgro_q, p, pp) { + __skb_unlink(p, nvgro_q); + + NAPI_GRO_CB(p)->data_offset = 0; + NAPI_GRO_CB(p)->frag0 = NULL; + NAPI_GRO_CB(p)->frag0_len = 0; + NAPI_GRO_CB(p)->same_flow = 1; + NAPI_GRO_CB(p)->flush_id = 0; + NAPI_GRO_CB(p)->count = 0; + NAPI_GRO_CB(p)->flush = skb_is_gso(p); + NAPI_GRO_CB(p)->free = 0; + NAPI_GRO_CB(p)->encap_mark = 0; + NAPI_GRO_CB(p)->recursion_counter = 0; + NAPI_GRO_CB(p)->is_fou = 0; + NAPI_GRO_CB(p)->is_atomic = 1; + NAPI_GRO_CB(p)->gro_remcsum_start = 0; + NAPI_GRO_CB(p)->csum_cnt = p->csum_level + 1; + NAPI_GRO_CB(p)->csum_valid = 0; + + inet_gro_receive(&h, p); + + if (p == f_skb) { + list_add(&p->list, &h); + NAPI_GRO_CB(p)->age = jiffies; + NAPI_GRO_CB(p)->last = p; + skb_shinfo(p)->gso_size = skb_gro_len(p); + } + + NAPI_GRO_CB(f_skb)->count++; + } + + skb_list_del_init(f_skb); + napi_gro_complete(napi, f_skb); +} + +/** + * @brief ether_update_fq_with_fs - Populates final queue with TTL = 1 packet + * + * @param[in] pdata: Ethernet driver private data + * @param[in] skb: Socket buffer. + */ +static inline void ether_update_fq_with_fs(struct ether_priv_data *pdata, + struct sk_buff *skb) +{ + if (!skb_queue_empty(&pdata->fq)) { + pdata->nvgro_dropped += pdata->fq.qlen; + __skb_queue_purge(&pdata->fq); + } + + /* queue skb to fq which has TTL = 1 */ + __skb_queue_tail(&pdata->fq, skb); + + pdata->expected_ip_id = NAPI_GRO_CB(skb)->flush_id + 1; +} + +/** + * @brief ether_get_skb_from_ip_id - Get SKB from MQ based on IPID. + * + * @param[in] mq: NVGRO packet out of order queue. + * @param[in] ip_id: IPv4 packet ID. + * + * @retval skb on Success + * @retval NULL on failure. + */ +static inline struct sk_buff *ether_get_skb_from_ip_id(struct sk_buff_head *mq, + u16 ip_id) +{ + struct sk_buff *p, *pp; + + skb_queue_walk_safe(mq, p, pp) { + if ((NAPI_GRO_CB(p)->flush_id) == ip_id) { + __skb_unlink(p, mq); + return p; + } + } + + return NULL; +} + +/** + * @brief ether_gro - Perform NVGRO packet merging. + * + * @param[in] fq: NVGRO packets sequence queue. + * @param[in] mq: NVGRO packet out of order queue. + * @param[in] napi: Driver NAPI instance. + */ +static inline void ether_gro(struct sk_buff_head *fq, struct sk_buff_head *mq, + struct napi_struct *napi) +{ + struct sk_buff *f_skb, *p; + u32 s_ip_id; + + if (skb_queue_empty(fq)) + return; + + f_skb = skb_peek_tail(fq); + + s_ip_id = NAPI_GRO_CB(f_skb)->flush_id; + + do { + s_ip_id++; + p = ether_get_skb_from_ip_id(mq, s_ip_id); + if (!p) + return; + + __skb_queue_tail(fq, p); + + if (NAPI_GRO_CB(p)->free == 2) + break; + } while (1); + + ether_gro_merge_complete(fq, napi); +} + +/** + * @brief ether_purge_q - Purge master queue based on packet age. + * + * @param[in] pdata: Ethernet private data. + */ +static inline void ether_purge_q(struct ether_priv_data *pdata) +{ + struct sk_buff *p, *pp; + + skb_queue_walk_safe(&pdata->mq, p, pp) { + if ((jiffies - NAPI_GRO_CB(p)->age) > + msecs_to_jiffies(pdata->pkt_age_msec)) { + __skb_unlink(p, &pdata->mq); + dev_consume_skb_any(p); + pdata->nvgro_dropped++; + } else { + return; + } + } +} + +/** + * @brief ether_nvgro_purge_timer - NVGRO purge timer handler. + * + * @param[in] t: Pointer to the timer. + */ +void ether_nvgro_purge_timer(struct timer_list *t) +{ + struct ether_priv_data *pdata = from_timer(pdata, t, nvgro_timer); + struct sk_buff *f_skb; + + if (atomic_read(&pdata->rx_state) == OSI_ENABLE) + return; + + atomic_set(&pdata->timer_state, OSI_ENABLE); + + ether_purge_q(pdata); + + f_skb = skb_peek(&pdata->fq); + if (!f_skb) + goto exit; + + if ((jiffies - NAPI_GRO_CB(f_skb)->age) > + msecs_to_jiffies(pdata->pkt_age_msec)) { + pdata->nvgro_dropped += pdata->fq.qlen; + __skb_queue_purge(&pdata->fq); + } + +exit: + atomic_set(&pdata->timer_state, OSI_DISABLE); + + mod_timer(&pdata->nvgro_timer, + jiffies + msecs_to_jiffies(pdata->nvgro_timer_intrvl)); +} + +/** + * @brief ether_do_nvgro - Perform NVGRO processing. + * + * @param[in] pdata: Ethernet private data. + * @param[in] napi: Ethernet driver NAPI instance. + * @param[in] skb: socket buffer + * + * @retval true on Success + * @retval false on failure. + */ +static bool ether_do_nvgro(struct ether_priv_data *pdata, + struct napi_struct *napi, + struct sk_buff *skb) +{ + struct udphdr *uh = (struct udphdr *)(skb->data + sizeof(struct iphdr)); + struct iphdr *iph = (struct iphdr *)skb->data; + struct sk_buff_head *mq = &pdata->mq; + struct ethhdr *ethh = eth_hdr(skb); + struct sock *sk = NULL; + + if (ethh->h_proto != htons(ETH_P_IP)) + return false; + + if (iph->protocol != IPPROTO_UDP) + return false; + + /* TODO: Hash based based queues selection */ + /* Socket look up with IPv4/UDP source/destination */ + sk = __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, uh->source, + iph->daddr, uh->dest, inet_iif(skb), + inet_sdif(skb), &udp_table, NULL); + if (!sk) + return false; + + /* Socket found but GRO not enabled on the socket - We don't care */ + if (!udp_sk(sk)->gro_enabled) + return false; + + /* Store IPID, TTL and age of skb inside per skb control block */ + NAPI_GRO_CB(skb)->flush_id = ntohs(iph->id); + NAPI_GRO_CB(skb)->free = (iph->ttl & (BIT(6) | BIT(7))) >> 6; + NAPI_GRO_CB(skb)->age = jiffies; + + while (atomic_read(&pdata->timer_state) == OSI_ENABLE) { + /* busyloop */ + }; + + atomic_set(&pdata->rx_state, OSI_ENABLE); + + if (NAPI_GRO_CB(skb)->free == 1) { + /* Update final queue with first segment */ + ether_update_fq_with_fs(pdata, skb); + goto exit; + } else { + if (pdata->expected_ip_id == NAPI_GRO_CB(skb)->flush_id) { + __skb_queue_tail(&pdata->fq, skb); + pdata->expected_ip_id = NAPI_GRO_CB(skb)->flush_id + 1; + + if (NAPI_GRO_CB(skb)->free == 2) + ether_gro_merge_complete(&pdata->fq, napi); + + goto exit; + } + } + + /* Add skb to the queue */ + __skb_queue_tail(mq, skb); + + /* Queue the packets until last segment received */ + if (NAPI_GRO_CB(skb)->free != 2) + goto exit; + + ether_gro(&pdata->fq, &pdata->mq, napi); + +exit: + atomic_set(&pdata->rx_state, OSI_DISABLE); + return true; +} +#endif + /** * @brief Handover received packet to network stack. * @@ -350,6 +616,11 @@ void osd_receive_packet(void *priv, struct osi_rx_ring *rx_ring, skb->dev = ndev; skb->protocol = eth_type_trans(skb, ndev); ndev->stats.rx_bytes += skb->len; +#ifdef ETHER_NVGRO + if ((ndev->features & NETIF_F_GRO) && + ether_do_nvgro(pdata, &rx_napi->napi, skb)) + goto done; +#endif if (likely(ndev->features & NETIF_F_GRO)) { napi_gro_receive(&rx_napi->napi, skb); } else { @@ -366,6 +637,9 @@ void osd_receive_packet(void *priv, struct osi_rx_ring *rx_ring, dev_kfree_skb_any(skb); } +#ifdef ETHER_NVGRO +done: +#endif ndev->stats.rx_packets++; rx_swcx->buf_virt_addr = NULL; rx_swcx->buf_phy_addr = 0; diff --git a/drivers/net/ethernet/nvidia/nvethernet/sysfs.c b/drivers/net/ethernet/nvidia/nvethernet/sysfs.c index 27288cd7..1c9a416f 100644 --- a/drivers/net/ethernet/nvidia/nvethernet/sysfs.c +++ b/drivers/net/ethernet/nvidia/nvethernet/sysfs.c @@ -2267,6 +2267,179 @@ static DEVICE_ATTR(ptp_sync, (S_IRUGO | S_IWUSR), ether_ptp_sync_show, ether_ptp_sync_store); +#ifdef ETHER_NVGRO +/** + * @brief Shows the current setting of NVGRO packet age threshold. + * + * @param[in] dev: Device data. + * @param[in] attr: Device attribute + * @param[in] buf: Buffer to store the current ptp sync method + */ +static ssize_t ether_nvgro_pkt_age_msec_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct net_device *ndev = (struct net_device *)dev_get_drvdata(dev); + struct ether_priv_data *pdata = netdev_priv(ndev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", pdata->pkt_age_msec); +} + +/** + * @brief Set the user setting of NVGRO packet age threshold. + * + * @param[in] dev: Device data. + * @param[in] attr: Device attribute + * @param[in] buf: Buffer which contains the user settings of MAC loopback + * @param[in] size: size of buffer + * + * @return size of buffer. + */ +static ssize_t ether_nvgro_pkt_age_msec_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct net_device *ndev = (struct net_device *)dev_get_drvdata(dev); + struct ether_priv_data *pdata = netdev_priv(ndev); + int ret; + + ret = kstrtou32(buf, 0, &pdata->pkt_age_msec); + if (ret < 0) { + dev_err(pdata->dev, + "Invalid nvgro pkt age msec input\n"); + return -EINVAL; + } + + return size; +} + +/** + * @brief Sysfs attribute for NVGRO packet age threshold + * + */ +static DEVICE_ATTR(nvgro_pkt_age_msec, 0644, + ether_nvgro_pkt_age_msec_show, + ether_nvgro_pkt_age_msec_store); + +/** + * @brief Shows the current setting of NVGRO purge timer interval.. + * + * @param[in] dev: Device data. + * @param[in] attr: Device attribute + * @param[in] buf: Buffer to store the current ptp sync method + */ +static ssize_t ether_nvgro_timer_interval_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct net_device *ndev = (struct net_device *)dev_get_drvdata(dev); + struct ether_priv_data *pdata = netdev_priv(ndev); + + return scnprintf(buf, PAGE_SIZE, "%u\n", pdata->nvgro_timer_intrvl); +} + +/** + * @brief Set the user setting of NVGRO purge timer interval. + * + * @param[in] dev: Device data. + * @param[in] attr: Device attribute + * @param[in] buf: Buffer which contains the user settings of MAC loopback + * @param[in] size: size of buffer + * + * @return size of buffer. + */ +static ssize_t ether_nvgro_timer_interval_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct net_device *ndev = (struct net_device *)dev_get_drvdata(dev); + struct ether_priv_data *pdata = netdev_priv(ndev); + int ret; + + ret = kstrtou32(buf, 0, &pdata->nvgro_timer_intrvl); + if (ret < 0) { + dev_err(pdata->dev, + "Invalid nvgro timer interval input\n"); + return -EINVAL; + } + + return size; +} + +/** + * @brief Sysfs attribute for NVGRO purge timer interval + * + */ +static DEVICE_ATTR(nvgro_timer_interval, 0644, + ether_nvgro_timer_interval_show, + ether_nvgro_timer_interval_store); + +/** + * @brief Shows NVGRO stats + * + * @param[in] dev: Device data. + * @param[in] attr: Device attribute + * @param[in] buf: Buffer to store the current ptp sync method + */ +static ssize_t ether_nvgro_stats_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct net_device *ndev = (struct net_device *)dev_get_drvdata(dev); + struct ether_priv_data *pdata = netdev_priv(ndev); + + return scnprintf(buf, PAGE_SIZE, "dropped = %llu\n", + pdata->nvgro_dropped); +} + +/** + * @brief Sysfs attribute for NVGRO stats. + * + */ +static DEVICE_ATTR(nvgro_stats, 0644, + ether_nvgro_stats_show, NULL); + +/** + * @brief Dumps NVGRO queues. + * + * @param[in] dev: Device data. + * @param[in] attr: Device attribute + * @param[in] buf: Buffer to store the current ptp sync method + */ +static ssize_t ether_nvgro_dump_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct net_device *ndev = (struct net_device *)dev_get_drvdata(dev); + struct ether_priv_data *pdata = netdev_priv(ndev); + struct sk_buff *p, *pp; + char *start = buf; + + buf += scnprintf(buf, PAGE_SIZE, "MQ: "); + skb_queue_walk_safe(&pdata->mq, p, pp) { + buf += scnprintf(buf, PAGE_SIZE, "skb %p TTL %d IPID %u\n", + p, NAPI_GRO_CB(p)->free, + NAPI_GRO_CB(p)->flush_id); + } + + buf += scnprintf(buf, PAGE_SIZE, "FQ: "); + skb_queue_walk_safe(&pdata->fq, p, pp) { + buf += scnprintf(buf, PAGE_SIZE, "skb %p TTL %d IPID %u\n", + p, NAPI_GRO_CB(p)->free, + NAPI_GRO_CB(p)->flush_id); + } + + return (buf - start); +} + +/** + * @brief Sysfs attribute for NVGRO queue dump. + * + */ +static DEVICE_ATTR(nvgro_dump, 0644, + ether_nvgro_dump_show, NULL); +#endif + /** * @brief Attributes for nvethernet sysfs */ @@ -2298,6 +2471,12 @@ static struct attribute *ether_sysfs_attrs[] = { &dev_attr_macsec_dbg_events.attr, #endif /* MACSEC_SUPPORT */ &dev_attr_uphy_gbe_mode.attr, +#ifdef ETHER_NVGRO + &dev_attr_nvgro_pkt_age_msec.attr, + &dev_attr_nvgro_timer_interval.attr, + &dev_attr_nvgro_stats.attr, + &dev_attr_nvgro_dump.attr, +#endif NULL };