nvethernetrm: mgbe: add support for RSS

This change programs 40byte Hash key and indirection table
(Hash table) (has DMA channel numbers) in MAC
Once packet received by MAC - 4-tuples will be extracted
from the packet and given to RSS hash engine. Hash function
will generate hash value by using 40byte key.
From hash value LSB bits used as index to RSS lookup table to
find out DMA channel number. If there is a match - packet is
routed to corresponding DMA channel. If there is no match -
packet will be dropped and error will be returned in receive desc.

Bug 200565647

Change-Id: Iffbb5a452f03278b3ba0bc061f09b43c7c994289
Signed-off-by: Bhadram Varka <vbhadram@nvidia.com>
Reviewed-on: https://git-master.nvidia.com/r/c/kernel/nvethernetrm/+/2263398
Reviewed-by: Rakesh Goyal <rgoyal@nvidia.com>

include/osi_core.h

@@ -181,6 +181,16 @@ typedef my_lint_64		nvel64_t;
 #define OSI_VLAN_ACTION_ADD	OSI_BIT(31)
 #define OSI_VLAN_ACTION_DEL	0x0U
 
+/**
+ * @addtogroup RSS related information
+ *
+ * @brief RSS hash key and table size.
+ * @{
+ */
+#define OSI_RSS_HASH_KEY_SIZE	40U
+#define OSI_RSS_MAX_TABLE_SIZE	128U
+/** @} */
+
 struct osi_core_priv_data;
 
 /**
@@ -650,6 +660,19 @@ struct osi_ptp_config {
 	nveu32_t ptp_rx_queue;
 };
 
+/**
+ * @brief osi_core_rss - Struture used to store RSS Hash key and table
+ * information.
+ */
+struct osi_core_rss {
+	/** Flag to represent to enable RSS or not */
+	unsigned int enable;
+	/** Array for storing RSS Hash key */
+	unsigned char key[OSI_RSS_HASH_KEY_SIZE];
+	/** Array for storing RSS Hash table */
+	unsigned int table[OSI_RSS_MAX_TABLE_SIZE];
+};
+
 /**
  * @brief Max num of MAC core registers to backup. It should be max of or >=
  * (EQOS_MAX_BAK_IDX=380, coreX,...etc) backup registers.
@@ -753,6 +776,8 @@ struct osi_core_priv_data {
 	unsigned short vid[VLAN_NUM_VID];
 	/** Count of number of VLAN filters in vid array */
 	unsigned short vlan_filter_cnt;
+	/** RSS core structure */
+	struct osi_core_rss rss;
 };
 
 /**
@@ -2191,4 +2216,17 @@ nve32_t osi_set_mdc_clk_rate(struct osi_core_priv_data *const osi_core,
 nve32_t osi_config_mac_loopback(struct osi_core_priv_data *const osi_core,
 				const nveu32_t lb_mode);
 #endif /* !OSI_STRIPPED_LIB */
+
+/**
+ * @brief osi_config_rss - Configuration of RSS.
+ *
+ * @param[in] osi_core: OSI core private data structure.
+ *
+ * @note
+ *	1) MAC and PHY should be init and started. see osi_start_mac()
+ *
+ * @retval 0 on success
+ * @retval -1 on failure.
+ */
+nve32_t osi_config_rss(struct osi_core_priv_data *const osi_core);
 #endif /* INCLUDED_OSI_CORE_H */

include/osi_dma.h

@@ -94,13 +94,14 @@
 #define OSI_PKT_CX_TSO			OSI_BIT(2)
 /** PTP packet */
 #define OSI_PKT_CX_PTP			OSI_BIT(3)
+/** Rx packet has RSS hash */
+#define OSI_PKT_CX_RSS			OSI_BIT(4)
 /** Valid packet */
 #define OSI_PKT_CX_VALID		OSI_BIT(10)
 /** Update Packet Length in Tx Desc3 */
 #define OSI_PKT_CX_LEN			OSI_BIT(11)
 /** IP CSUM packet */
 #define OSI_PKT_CX_IP_CSUM		OSI_BIT(12)
-
 /** @} */
 
 /**
@@ -182,6 +183,19 @@
 
 /** @} */
 
+
+/**
+ * @addtogroup RSS-HASH type
+ *
+ * @brief Macros to represent to type of packet for hash stored in receive packet
+ * context.
+ * @{
+ */
+#define OSI_RX_PKT_HASH_TYPE_L2	0x1U
+#define OSI_RX_PKT_HASH_TYPE_L3	0x2U
+#define OSI_RX_PKT_HASH_TYPE_L4	0x3U
+/** @} */
+
 /**
  * @addtogroup OSI-INTR OSI DMA interrupt handling macros.
  *
@@ -270,6 +284,10 @@ struct osi_rx_pkt_cx {
 	nveu32_t vlan_tag;
 	/** Length of received packet */
 	nveu32_t pkt_len;
+	/** Stores received packet hash */
+	nveu32_t rx_hash;
+	/** Store type of packet for which hash carries at rx_hash */
+	nveu32_t rx_hash_type;
 	/** TS in nsec for the received packet */
 	nveul64_t ns;
 };

include/osi_dma_txrx.h

@@ -53,5 +53,4 @@
 #define DECR_RX_DESC_INDEX(idx, i) ((idx) = ((idx) - (i)) & (RX_DESC_CNT - 1U))
 #endif /* !OSI_STRIPPED_LIB */
 /** @} */
-
 #endif /* INCLUDED_OSI_DMA_TXRX_H */

osi/core/core_local.h

@@ -201,6 +201,8 @@ struct core_ops {
 	/** Called to get HW features */
 	nve32_t (*get_hw_features)(struct osi_core_priv_data *const osi_core,
 				   struct osi_hw_features *hw_feat);
+	/** Called to configure RSS for MAC */
+	nve32_t (*config_rss)(struct osi_core_priv_data *osi_core);
 };
 
 

osi/core/mgbe_core.c

@@ -1779,6 +1779,126 @@ static nve32_t mgbe_configure_mtl_queue(nveu32_t qinx,
 	return 0;
 }
 
+/**
+ * @brief mgbe_rss_write_reg - Write into RSS registers
+ *
+ * Algorithm: Programes RSS hash table or RSS hash key.
+ *
+ * @param[in] addr: MAC base address
+ * @param[in] idx: Hash table or key index
+ * @param[in] value: Value to be programmed in RSS data register.
+ * @param[in] is_key: To represent passed value key or table data.
+ *
+ * @note MAC has to be out of reset.
+ *
+ * @retval 0 on success
+ * @retval -1 on failure.
+ */
+static int mgbe_rss_write_reg(struct osi_core_priv_data *osi_core,
+			      unsigned int idx,
+			      unsigned int value,
+			      unsigned int is_key)
+{
+	unsigned char *addr = (unsigned char *)osi_core->base;
+	unsigned int retry = 100;
+	unsigned int ctrl = 0;
+	unsigned int count = 0;
+	int cond = 1;
+
+	/* data into RSS Lookup Table or RSS Hash Key */
+	osi_writel(value, addr + MGBE_MAC_RSS_DATA);
+
+	if (is_key == OSI_ENABLE) {
+		ctrl |= MGBE_MAC_RSS_ADDR_ADDRT;
+	}
+
+	ctrl |= idx << MGBE_MAC_RSS_ADDR_RSSIA_SHIFT;
+	ctrl |= MGBE_MAC_RSS_ADDR_OB;
+	ctrl &= ~MGBE_MAC_RSS_ADDR_CT;
+	osi_writel(ctrl, addr + MGBE_MAC_RSS_ADDR);
+
+	/* poll for write operation to complete */
+	while (cond == 1) {
+		if (count > retry) {
+			OSI_CORE_ERR(OSI_NULL, OSI_LOG_ARG_HW_FAIL,
+				     "Failed to update RSS Hash key or table\n",
+				     0ULL);
+			return -1;
+		}
+
+		count++;
+
+		value = osi_readl(addr + MGBE_MAC_RSS_ADDR);
+		if ((value & MGBE_MAC_RSS_ADDR_OB) == OSI_NONE) {
+			cond = 0;
+		} else {
+			osi_core->osd_ops.udelay(100);
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * @brief mgbe_config_rss - Configure RSS
+ *
+ * Algorithm: Programes RSS hash table or RSS hash key.
+ *
+ * @param[in] osi_core: OSI core private data.
+ *
+ * @note MAC has to be out of reset.
+ *
+ * @retval 0 on success
+ * @retval -1 on failure.
+ */
+static int mgbe_config_rss(struct osi_core_priv_data *osi_core)
+{
+	unsigned char *addr = (unsigned char *)osi_core->base;
+	unsigned int value = 0;
+	unsigned int i = 0, j = 0;
+	int ret = 0;
+
+	if (osi_core->rss.enable == OSI_DISABLE) {
+		/* RSS not supported */
+		return 0;
+	}
+
+	/* No need to enable RSS for single Queue */
+	if (osi_core->num_mtl_queues == 1U) {
+		return 0;
+	}
+
+	/* Program the hash key */
+	for (i = 0; i < OSI_RSS_HASH_KEY_SIZE; i += 4U) {
+		value = ((unsigned int)osi_core->rss.key[i] |
+			 (unsigned int)osi_core->rss.key[i + 1U] << 8U |
+			 (unsigned int)osi_core->rss.key[i + 2U] << 16U |
+			 (unsigned int)osi_core->rss.key[i + 3U] << 24U);
+		ret = mgbe_rss_write_reg(osi_core, j, value, OSI_ENABLE);
+		if (ret < 0) {
+			return ret;
+		}
+		j++;
+	}
+
+	/* Program Hash table */
+	for (i = 0; i < OSI_RSS_MAX_TABLE_SIZE; i++) {
+		ret = mgbe_rss_write_reg(osi_core, i, osi_core->rss.table[i],
+					 OSI_NONE);
+		if (ret < 0) {
+			return ret;
+		}
+	}
+
+	/* Enable RSS */
+	value = osi_readl(addr + MGBE_MAC_RSS_CTRL);
+	value |= MGBE_MAC_RSS_CTRL_UDP4TE | MGBE_MAC_RSS_CTRL_TCP4TE |
+		 MGBE_MAC_RSS_CTRL_IP2TE | MGBE_MAC_RSS_CTRL_RSSE;
+	osi_writel(value, addr + MGBE_MAC_RSS_CTRL);
+
+	return 0;
+}
+
 /**
  * @brief mgbe_config_flow_control - Configure MAC flow control settings
  *
@@ -1857,8 +1977,11 @@ static int mgbe_config_flow_control(struct osi_core_priv_data *const osi_core,
  * @param[in] osi_core: OSI core private data structure.
  *
  * @note MAC has to be out of reset.
+ *
+ * @retval 0 on success
+ * @retval -1 on failure.
  */
-static void mgbe_configure_mac(struct osi_core_priv_data *osi_core)
+static int mgbe_configure_mac(struct osi_core_priv_data *osi_core)
 {
 	nveu32_t value;
 
@@ -1968,6 +2091,9 @@ static void mgbe_configure_mac(struct osi_core_priv_data *osi_core)
 		}
 	}
 	/* TODO: USP (user Priority) to RxQ Mapping */
+
+	/* RSS cofiguration */
+	return mgbe_config_rss(osi_core);
 }
 
 /**
@@ -2125,19 +2251,22 @@ static nve32_t mgbe_core_init(struct osi_core_priv_data *osi_core,
 	value = osi_readl((nveu8_t *)osi_core->base +
 			  MGBE_MTL_RXQ_DMA_MAP0);
 	value |= MGBE_RXQ_TO_DMA_CHAN_MAP0;
-	osi_writel(value, (nveu8_t *)osi_core->base +
+	value |= MGBE_RXQ_TO_DMA_MAP_DDMACH;
+	osi_writel(value, (unsigned char *)osi_core->base +
 		   MGBE_MTL_RXQ_DMA_MAP0);
 
 	value = osi_readl((nveu8_t *)osi_core->base +
 			  MGBE_MTL_RXQ_DMA_MAP1);
 	value |= MGBE_RXQ_TO_DMA_CHAN_MAP1;
-	osi_writel(value, (nveu8_t *)osi_core->base +
+	value |= MGBE_RXQ_TO_DMA_MAP_DDMACH;
+	osi_writel(value, (unsigned char *)osi_core->base +
 		   MGBE_MTL_RXQ_DMA_MAP1);
 
 	value = osi_readl((nveu8_t *)osi_core->base +
 			  MGBE_MTL_RXQ_DMA_MAP2);
 	value |= MGBE_RXQ_TO_DMA_CHAN_MAP2;
-	osi_writel(value, (nveu8_t *)osi_core->base +
+	value |= MGBE_RXQ_TO_DMA_MAP_DDMACH;
+	osi_writel(value, (unsigned char *)osi_core->base +
 		   MGBE_MTL_RXQ_DMA_MAP2);
 
 	/* TODO: DCS enable */
@@ -2161,7 +2290,10 @@ static nve32_t mgbe_core_init(struct osi_core_priv_data *osi_core,
 	}
 
 	/* configure MGBE MAC HW */
-	mgbe_configure_mac(osi_core);
+	ret = mgbe_configure_mac(osi_core);
+	if (ret < 0) {
+		return ret;
+	}
 
 	/* configure MGBE DMA */
 	mgbe_configure_dma(osi_core->base);
@@ -3303,4 +3435,5 @@ void mgbe_init_core_ops(struct core_ops *ops)
 	ops->reset_mmc = mgbe_reset_mmc;
 	ops->configure_eee = mgbe_configure_eee;
 	ops->get_hw_features = mgbe_get_hw_features;
+	ops->config_rss = mgbe_config_rss;
 };

osi/core/mgbe_core.h

@@ -97,6 +97,9 @@
 #define MGBE_MAC_ARPPA			0x0c10
 #define MGBE_MAC_L3L4_ADDR_CTR		0x0C00
 #define MGBE_MAC_L3L4_DATA		0x0C04
+#define MGBE_MAC_RSS_CTRL		0x0C80
+#define MGBE_MAC_RSS_ADDR		0x0C88
+#define MGBE_MAC_RSS_DATA		0x0C8C
 /** @} */
 
 /**
@@ -344,7 +347,14 @@
 #define MGBE_MAC_LPI_CSR_LPITXA			OSI_BIT(19)
 #define MGBE_MAC_LPI_CSR_PLS			OSI_BIT(17)
 #define MGBE_MAC_LPI_CSR_LPIEN			OSI_BIT(16)
-
+#define MGBE_MAC_RSS_CTRL_RSSE			OSI_BIT(0)
+#define MGBE_MAC_RSS_CTRL_IP2TE			OSI_BIT(1)
+#define MGBE_MAC_RSS_CTRL_TCP4TE		OSI_BIT(2)
+#define MGBE_MAC_RSS_CTRL_UDP4TE		OSI_BIT(3)
+#define MGBE_MAC_RSS_ADDR_ADDRT			OSI_BIT(2)
+#define MGBE_MAC_RSS_ADDR_RSSIA_SHIFT		8U
+#define MGBE_MAC_RSS_ADDR_OB			OSI_BIT(0)
+#define MGBE_MAC_RSS_ADDR_CT			OSI_BIT(1)
 /* DMA SBUS */
 #define MGBE_DMA_SBUS_BLEN8			OSI_BIT(2)
 #define MGBE_DMA_SBUS_BLEN16			OSI_BIT(3)
@@ -362,6 +372,7 @@
 #define MGBE_RXQ_TO_DMA_CHAN_MAP1		0x07060504U
 #define MGBE_RXQ_TO_DMA_CHAN_MAP2		0x0B0A0908U
 #define MGBE_RXQ_TO_DMA_CHAN_MAP3		0x0F0E0D0CU
+#define MGBE_RXQ_TO_DMA_MAP_DDMACH		0x80808080U
 #define MGBE_MTL_TXQ_SIZE_SHIFT			16U
 #define MGBE_MTL_RXQ_SIZE_SHIFT			16U
 #define MGBE_MAC_RMCR_GPSL_MSK			0x3FFF0000U

osi/core/osi_core.c

@@ -982,6 +982,15 @@ nve32_t osi_set_mdc_clk_rate(struct osi_core_priv_data *const osi_core,
 	return 0;
 }
 
+int osi_config_rss(struct osi_core_priv_data *const osi_core)
+{
+	if (validate_args(osi_core) < 0) {
+		return -1;
+	}
+
+	return ops_p->config_rss(osi_core);
+}
+
 nve32_t osi_config_mac_loopback(struct osi_core_priv_data *const osi_core,
 				const nveu32_t lb_mode)
 {

osi/dma/dma_local.h

@@ -98,6 +98,9 @@ struct desc_ops {
 	/** Called to get rx VLAN from descriptor */
 	void (*get_rx_vlan)(struct osi_rx_desc *rx_desc,
 			    struct osi_rx_pkt_cx *rx_pkt_cx);
+	/** Called to get rx HASH from descriptor */
+	void (*get_rx_hash)(struct osi_rx_desc *rx_desc,
+			    struct osi_rx_pkt_cx *rx_pkt_cx);
 };
 
 /**

osi/dma/eqos_desc.c

@@ -160,9 +160,26 @@ static void eqos_get_rx_csum(struct osi_rx_desc *rx_desc,
 	}
 }
 
+/**
+ * @brief eqos_get_rx_hash - Get Rx packet hash from descriptor if valid
+ *
+ * Algorithm: This routine will be invoked by OSI layer itself to get received
+ * packet Hash from descriptor if RSS hash is valid and it also sets the type
+ * of RSS hash.
+ *
+ * @param[in] rx_desc: Rx Descriptor.
+ * @param[in] rx_pkt_cx: Per-Rx packet context structure
+ */
+static void eqos_get_rx_hash(struct osi_rx_desc *rx_desc,
+			     struct osi_rx_pkt_cx *rx_pkt_cx)
+{
+}
+
+
 void eqos_init_desc_ops(struct desc_ops *d_ops)
 {
 	d_ops->get_rx_csum = eqos_get_rx_csum;
 	d_ops->update_rx_err_stats = eqos_update_rx_err_stats;
 	d_ops->get_rx_vlan = eqos_get_rx_vlan;
+	d_ops->get_rx_hash = eqos_get_rx_hash;
 }

osi/dma/hw_desc.h

@@ -49,6 +49,12 @@
 #define RDES3_LT_DVT		(OSI_BIT(16) | OSI_BIT(18))
 #define RDES3_RS0V		OSI_BIT(25)
 #define RDES3_RS1V		OSI_BIT(26)
+#define RDES3_RSV		OSI_BIT(26)
+#define RDES3_L34T		0x00F00000U
+#define RDES3_L34T_IPV4_TCP	OSI_BIT(20)
+#define RDES3_L34T_IPV4_UDP	OSI_BIT(21)
+#define RDES3_L34T_IPV6_TCP	(OSI_BIT(23) | OSI_BIT(20))
+#define RDES3_L34T_IPV6_UDP	(OSI_BIT(23) | OSI_BIT(21))
 #define RDES0_OVT		0x0000FFFFU
 #define RDES1_TSA		OSI_BIT(14)
 #define RDES1_TD		OSI_BIT(15)

osi/dma/mgbe_desc.c

@@ -96,9 +96,46 @@ static void mgbe_get_rx_csum(struct osi_rx_desc *rx_desc,
 	}
 }
 
+/**
+ * @brief mgbe_get_rx_hash - Get Rx packet hash from descriptor if valid
+ *
+ * Algorithm: This routine will be invoked by OSI layer itself to get received
+ * packet Hash from descriptor if RSS hash is valid and it also sets the type
+ * of RSS hash.
+ *
+ * @param[in] rx_desc: Rx Descriptor.
+ * @param[in] rx_pkt_cx: Per-Rx packet context structure
+ */
+static void mgbe_get_rx_hash(struct osi_rx_desc *rx_desc,
+			     struct osi_rx_pkt_cx *rx_pkt_cx)
+{
+	unsigned int pkt_type = rx_desc->rdes3 & RDES3_L34T;
+
+	if ((rx_desc->rdes3 & RDES3_RSV) != RDES3_RSV) {
+		return;
+	}
+
+	switch (pkt_type) {
+	case RDES3_L34T_IPV4_TCP:
+	case RDES3_L34T_IPV4_UDP:
+	case RDES3_L34T_IPV6_TCP:
+	case RDES3_L34T_IPV6_UDP:
+		rx_pkt_cx->rx_hash_type = OSI_RX_PKT_HASH_TYPE_L4;
+		break;
+	default:
+		rx_pkt_cx->rx_hash_type = OSI_RX_PKT_HASH_TYPE_L3;
+		break;
+	}
+
+	/* Get Rx hash from RDES1 RSSH */
+	rx_pkt_cx->rx_hash = rx_desc->rdes1;
+	rx_pkt_cx->flags |= OSI_PKT_CX_RSS;
+}
+
 void mgbe_init_desc_ops(struct desc_ops *d_ops)
 {
         d_ops->get_rx_csum = mgbe_get_rx_csum;
 	d_ops->update_rx_err_stats = mgbe_update_rx_err_stats;
 	d_ops->get_rx_vlan = mgbe_get_rx_vlan;
+	d_ops->get_rx_hash = mgbe_get_rx_hash;
 }

osi/dma/osi_dma_local.h

@@ -41,6 +41,9 @@ struct desc_ops {
 	/** Called to get rx VLAN from descriptor */
 	void (*get_rx_vlan)(struct osi_rx_desc *rx_desc,
 			    struct osi_rx_pkt_cx *rx_pkt_cx);
+	/** Called to get rx HASH from descriptor */
+	void (*get_rx_hash)(struct osi_rx_desc *rx_desc,
+			    struct osi_rx_pkt_cx *rx_pkt_cx);
 };
 
 /**

osi/dma/osi_dma_txrx.c

@@ -334,6 +334,9 @@ nve32_t osi_process_rx_completions(struct osi_dma_priv_data *osi_dma,
 			/* Get Rx VLAN from descriptor */
 			d_ops.get_rx_vlan(rx_desc, rx_pkt_cx);
 
+			/* get_rx_hash for RSS */
+			d_ops.get_rx_hash(rx_desc, rx_pkt_cx);
+
 			context_desc = rx_ring->rx_desc + rx_ring->cur_rx_idx;
 			/* Get rx time stamp */
 			ret = get_rx_hwstamp(osi_dma, rx_desc, context_desc,