Increase default mbuf size and code simplification/cleanup

mbuf size was setup to achieve the best performance i.e.
using the smallest mbuf and not segmenting packets.
However this resulted in complex code, much dependent of the way
the pmd are working e.g. a change(fix) in recent dpdk i40e
implementation caused a 1782 (=1518+8+256) bytes mbuf to be too
small to hold a 1518 bytes packets.
Hence this change simplifies the mbuf size selection at the price
of a potential decreases in performance - as more memory is now used.
Except if jumbo frames are used, the mbuf size will now be the same
for all modes. The packets will not be segmented except if jumbo
frames are enabled.
If jumbo frames are enabled, packets are by default segmented, except
if the mbuf size is configured big enough in the config file.

Change-Id: I222fcac7a65c0d221d5d422f419deb9c0f864172
Signed-off-by: Xavier Simonart <[email protected]>
Signed-off-by: Deepak S <[email protected]>

VNFs/DPPD-PROX/defaults.c

@@ -54,7 +54,8 @@ static const struct rte_eth_conf default_port_conf = {
 		.jumbo_frame    = 0, /* Jumbo frame support disabled */
 		.hw_strip_crc   = 1, /* CRC stripped by hardware --- always set to 1 in VF */
 		.hw_vlan_extend = 0,
-		.mq_mode        = 0
+		.mq_mode        = 0,
+		.max_rx_pkt_len = PROX_MTU + ETHER_HDR_LEN + ETHER_CRC_LEN
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
@@ -164,7 +165,6 @@ void set_task_defaults(struct prox_cfg* prox_cfg, struct lcore_cfg* lcore_cfg_in
 			targ->tunnel_hop_limit = 3;
 			targ->ctrl_freq = 1000;
 			targ->lb_friend_core = 0xFF;
-			targ->mbuf_size = MBUF_SIZE;
 			targ->n_pkts = 1024*64;
 			targ->runtime_flags |= TASK_TX_CRC;
 			targ->accuracy_limit_nsec = 5000;

VNFs/DPPD-PROX/defaults.h

@@ -41,7 +41,24 @@ void set_port_defaults(void);
 #define MAX_RSS_QUEUE_BITS      9
 
 #define PROX_VLAN_TAG_SIZE	4
-#define MBUF_SIZE (ETHER_MAX_LEN + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM +  2 * PROX_VLAN_TAG_SIZE)
+
+/* MBUF_SIZE can be configured based on the following:
+   - If only one segment is used ETH_TXQ_FLAGS_NOMULTSEGS can be used resulting
+     in vector mode used for transmission hence higher performance
+   - Only one segment is used by the rx function if the mbuf size is big enough
+   - Bigger mbufs result in more memory used, hence slighly lower performance (DTLB misses)
+   - Selecting the smaller mbuf is not obvious as pmds might behave slighly differently:
+     - on ixgbe a 1526 + 256 mbuf size will cause any packets bigger than 1024 bytes to be segmented
+     - on i40e a 1526 + 256 mbuf size will cause any packets bigger than 1408 bytes to be segmented
+     - other pmds might have additional requirements
+   As the performance decrease due to the usage of bigger mbuf is not very important, we prefer
+   here to use  the same, bigger, mbuf size for all pmds, making the code easier to support.
+   An mbuf size of 2048 + 128 + 128 + 8 can hold a 2048 packet, and only one segment will be used
+   except if jumbo frames are enabled. +8 (VLAN) is needed for i40e (and maybe other pmds).
+   TX_MBUF_SIZE is used for when transmitting only: in this case the mbuf size can be smaller.
+*/
+#define MBUF_SIZE (2048 + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM + 2 * PROX_VLAN_TAG_SIZE)
+#define TX_MBUF_SIZE (ETHER_MAX_LEN + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM +  2 * PROX_VLAN_TAG_SIZE)
 
 #define PROX_MTU   ETHER_MAX_LEN - ETHER_HDR_LEN - ETHER_CRC_LEN
 

VNFs/DPPD-PROX/handle_cgnat.c

@@ -973,12 +973,11 @@ static struct task_init task_init_nat = {
 	.init = init_task_nat,
 	.handle = handle_nat_bulk,
 #ifdef SOFT_CRC
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS|TASK_FEATURE_ROUTING|TASK_FEATURE_ZERO_RX,
+	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_ROUTING|TASK_FEATURE_ZERO_RX,
 #else
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS|TASK_FEATURE_ROUTING|TASK_FEATURE_ZERO_RX,
+	.flag_features = TASK_FEATURE_ROUTING|TASK_FEATURE_ZERO_RX,
 #endif
 	.size = sizeof(struct task_nat),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_nat(void)

VNFs/DPPD-PROX/handle_esp.c

@@ -691,21 +691,19 @@ static int handle_esp_dec_bulk(struct task_base *tbase, struct rte_mbuf **mbufs,
 }
 
 struct task_init task_init_esp_enc = {
-	.mode = ESP_ENC,
-	.mode_str = "esp_enc",
-	.init = init_task_esp_enc,
-	.handle = handle_esp_enc_bulk,
-	.size = sizeof(struct task_esp_enc),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM
+        .mode = ESP_ENC,
+        .mode_str = "esp_enc",
+        .init = init_task_esp_enc,
+        .handle = handle_esp_enc_bulk,
+        .size = sizeof(struct task_esp_enc),
 };
 
 struct task_init task_init_esp_dec = {
-	.mode = ESP_ENC,
-	.mode_str = "esp_dec",
-	.init = init_task_esp_dec,
-	.handle = handle_esp_dec_bulk,
-	.size = sizeof(struct task_esp_dec),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM
+        .mode = ESP_ENC,
+        .mode_str = "esp_dec",
+        .init = init_task_esp_dec,
+        .handle = handle_esp_dec_bulk,
+        .size = sizeof(struct task_esp_dec),
 };
 
 __attribute__((constructor)) static void reg_task_esp_enc(void)

VNFs/DPPD-PROX/handle_gen.c

@@ -1005,15 +1005,20 @@ static struct rte_mempool *task_gen_create_mempool(struct task_args *targ, uint1
 	const int sock_id = rte_lcore_to_socket_id(targ->lconf->id);
 
 	name[0]++;
-	uint32_t mbuf_size = MBUF_SIZE;
+	uint32_t mbuf_size = TX_MBUF_SIZE;
 	if (max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM > mbuf_size)
 		mbuf_size = max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
+	plog_info("\t\tCreating mempool with name '%s'\n", name);
 	ret = rte_mempool_create(name, targ->nb_mbuf - 1, mbuf_size,
 				 targ->nb_cache_mbuf, sizeof(struct rte_pktmbuf_pool_private),
 				 rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
 				 sock_id, 0);
 	PROX_PANIC(ret == NULL, "Failed to allocate dummy memory pool on socket %u with %u elements\n",
 		   sock_id, targ->nb_mbuf - 1);
+
+        plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", ret,
+                  targ->nb_mbuf - 1, mbuf_size, targ->nb_cache_mbuf, sock_id);
+
 	return ret;
 }
 
@@ -1318,7 +1323,7 @@ static struct task_init task_init_gen = {
 #ifdef SOFT_CRC
 	// For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
 	// vector mode is used by DPDK, resulting (theoretically) in higher performance.
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 #else
 	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
 #endif
@@ -1335,7 +1340,7 @@ static struct task_init task_init_gen_l3 = {
 #ifdef SOFT_CRC
 	// For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
 	// vector mode is used by DPDK, resulting (theoretically) in higher performance.
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 #else
 	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
 #endif
@@ -1350,7 +1355,7 @@ static struct task_init task_init_gen_pcap = {
 	.start = start_pcap,
 	.early_init = init_task_gen_early,
 #ifdef SOFT_CRC
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 #else
 	.flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
 #endif

VNFs/DPPD-PROX/handle_genl4.c

@@ -845,7 +845,7 @@ static void init_task_gen(struct task_base *tbase, struct task_args *targ)
 	static char name[] = "server_mempool";
 	name[0]++;
 	task->mempool = rte_mempool_create(name,
-					   4*1024 - 1, MBUF_SIZE,
+					   4*1024 - 1, TX_MBUF_SIZE,
 					   targ->nb_cache_mbuf,
 					   sizeof(struct rte_pktmbuf_pool_private),
 					   rte_pktmbuf_pool_init, NULL,
@@ -959,7 +959,7 @@ static void init_task_gen_client(struct task_base *tbase, struct task_args *targ
 	const uint32_t socket = rte_lcore_to_socket_id(targ->lconf->id);
 	name[0]++;
 	task->mempool = rte_mempool_create(name,
-					   4*1024 - 1, MBUF_SIZE,
+					   4*1024 - 1, TX_MBUF_SIZE,
 					   targ->nb_cache_mbuf,
 					   sizeof(struct rte_pktmbuf_pool_private),
 					   rte_pktmbuf_pool_init, NULL,
@@ -1118,7 +1118,6 @@ static struct task_init task_init_gen1 = {
 	.stop = stop_task_gen_server,
 	.flag_features = TASK_FEATURE_ZERO_RX,
 	.size = sizeof(struct task_gen_server),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 static struct task_init task_init_gen2 = {
@@ -1129,7 +1128,6 @@ static struct task_init task_init_gen2 = {
 	.stop = stop_task_gen_client,
 	.flag_features = TASK_FEATURE_ZERO_RX,
 	.size = sizeof(struct task_gen_client),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_gen(void)

VNFs/DPPD-PROX/handle_l2fwd.c

@@ -112,9 +112,8 @@ static struct task_init task_init_l2fwd = {
 	.mode_str = "l2fwd",
 	.init = init_task_l2fwd,
 	.handle = handle_l2fwd_bulk,
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 	.size = sizeof(struct task_l2fwd),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_l2fwd(void)

VNFs/DPPD-PROX/handle_mirror.c

@@ -139,19 +139,17 @@ static struct task_init task_init_mirror = {
 	.mode_str = "mirror",
 	.init = init_task_mirror,
 	.handle = handle_mirror_bulk,
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS | TASK_FEATURE_TXQ_FLAGS_REFCOUNT,
+	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_REFCOUNT,
 	.size = sizeof(struct task_mirror),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 static struct task_init task_init_mirror2 = {
 	.mode_str = "mirror",
 	.sub_mode_str = "copy",
 	.init = init_task_mirror_copy,
 	.handle = handle_mirror_bulk_copy,
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 	.size = sizeof(struct task_mirror),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_mirror(void)

VNFs/DPPD-PROX/handle_nat.c

@@ -182,12 +182,11 @@ static struct task_init task_init_nat = {
 	.init = init_task_nat,
 	.handle = handle_nat_bulk,
 #ifdef SOFT_CRC
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 #else
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = 0,
 #endif
 	.size = sizeof(struct task_nat),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_nat(void)

VNFs/DPPD-PROX/handle_nop.c

@@ -22,9 +22,8 @@ static struct task_init task_init_nop_thrpt_opt = {
 	.init = NULL,
 	.handle = handle_nop_bulk,
 	.thread_x = thread_nop,
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS|TASK_FEATURE_THROUGHPUT_OPT|TASK_FEATURE_MULTI_RX,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_THROUGHPUT_OPT|TASK_FEATURE_MULTI_RX,
 	.size = sizeof(struct task_nop),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 static struct task_init task_init_nop_lat_opt = {
@@ -33,9 +32,8 @@ static struct task_init task_init_nop_lat_opt = {
 	.init = NULL,
 	.handle = handle_nop_bulk,
 	.thread_x = thread_nop,
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS|TASK_FEATURE_MULTI_RX,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_MULTI_RX,
 	.size = sizeof(struct task_nop),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 static struct task_init task_init_none;

VNFs/DPPD-PROX/handle_swap.c

@@ -246,19 +246,17 @@ static struct task_init task_init_swap = {
 	.mode_str = "swap",
 	.init = init_task_swap,
 	.handle = handle_swap_bulk,
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 	.size = sizeof(struct task_swap),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 static struct task_init task_init_swap_arp = {
 	.mode_str = "swap",
 	.sub_mode_str = "l3",
 	.init = init_task_swap,
 	.handle = handle_swap_bulk,
-	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
+	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
 	.size = sizeof(struct task_swap),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_swap(void)

VNFs/DPPD-PROX/handle_tsc.c

@@ -40,9 +40,8 @@ static struct task_init task_init = {
 	.mode_str = "tsc",
 	.init = NULL,
 	.handle = handle_bulk_tsc,
-	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS|TASK_FEATURE_THROUGHPUT_OPT,
+	.flag_features = TASK_FEATURE_NEVER_DISCARDS|TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_THROUGHPUT_OPT,
 	.size = sizeof(struct task_tsc),
-	.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
 };
 
 __attribute__((constructor)) static void reg_task_nop(void)