Fix TCP session establishment not completing

* Test with eUPF

.dockerignore

@@ -6,4 +6,5 @@ docs/
 
 *.o
 *.pcap
-*.log
+*.log
+*.pcapng

.gitignore

@@ -7,5 +7,6 @@
 *.ll
 
 *.pcap
+*.pcapng
 
 *.log

.vscode/settings.json

@@ -7,6 +7,7 @@
         "unistd.h": "c",
         "signal.h": "c",
         "gtpu.h": "c",
-        "tc-gtpu.skel.h": "c"
+        "tc-gtpu.skel.h": "c",
+        "ios": "c"
     }
 }

NOTES.md

@@ -28,6 +28,10 @@ cat /sys/kernel/debug/tracing/trace_pipe
 
 Build with Dockerfile
 
+```bash
+docker buildx build --platform=linux/amd64 -t tariromukute/tc-gtpu:latest -f Dockerfile .
+```
+
 Either:
 1. create container and the mount the debugfs inside the container.
 
@@ -184,6 +188,13 @@ ethtool --offload  <iface_name> rx off tx off
 
 # Check for TCP related errors
 netstat -s
+```
+
+```bash
+sysctl net.ipv4.tcp_timestamps
+
+sysctl -w net.ipv4.tcp_timestamps=0
+
 ```
 ## Useful Resources
 
@@ -196,4 +207,6 @@ netstat -s
 - https://lore.kernel.org/bpf/[email protected]/
 - https://github.com/siemens/edgeshark?tab=readme-ov-file#siemens-industrial-edge
 - https://www.dasblinkenlichten.com/working-with-tc-on-linux-systems/
-- https://www.alibabacloud.com/blog/why-are-linux-kernel-protocol-stacks-dropping-syn-packets_595251
+- https://www.alibabacloud.com/blog/why-are-linux-kernel-protocol-stacks-dropping-syn-packets_595251
+- https://arstechnica.com/civis/threads/a-possibly-simple-sniffer-trace-question-psh-ack.343792/
+- https://blogs.oracle.com/linux/post/notes-on-bpf-7-bpf-tc-and-generic-segmentation-offload

docker-compose/docker-compose-pfcp-eupf.yaml

@@ -13,6 +13,7 @@ services:
                 ipv4_address: 192.168.70.131
     tc-gtpu-tnl:
         privileged: true
+        platform: linux/amd64
         container_name: "tc-gtpu-tnl"
         image: tariromukute/tc-gtpu:latest
         command: tail -f /dev/null
@@ -32,25 +33,36 @@ services:
             public_net:
                 ipv4_address: 192.168.70.130
     edgecomllc-eupf:
+        platform: linux/amd64
         container_name: "edgecomllc-eupf"
         image: ghcr.io/edgecomllc/eupf:main
+        entrypoint:
+        - /bin/sh
+        - -c
+        - |
+            ip route del default;
+            ip route add default via 192.168.72.135 dev eth0 &&
+            sh /app/bin/entrypoint.sh
         environment:
             - UPF_INTERFACE_NAME=eth1,eth0
             - UPF_XDP_ATTACH_MODE=generic
-            - UPF_API_ADDRESS=:8081
+            - UPF_API_ADDRESS=:8080
             - UPF_PFCP_ADDRESS=:8805
             - UPF_METRICS_ADDRESS=:9091
             - UPF_PFCP_NODE_ID=192.168.70.134
             - UPF_N3_ADDRESS=192.168.70.134
+            - UPF_UEIP_POOL=12.1.1.0/24
+            - UPF_LOGGING_LEVEL=debug
         cap_add:
             - NET_ADMIN
             - SYS_ADMIN
             - SYS_RESOURCE # setrlimit
         # cap_drop:
         #     - ALL
         ports:
-            - "8880:8080"
-            - "9090:9090"
+            - "127.0.0.1:8081:8081"
+            - "127.0.0.1:8880:8080"
+            - "127.0.0.1:9090:9090"
         sysctls:
             - net.ipv4.conf.all.forwarding=1
         privileged: true
@@ -61,13 +73,14 @@ services:
                 ipv4_address: 192.168.72.134
     oai-ext-dn:
         privileged: true
+        platform: linux/amd64
         init: true
         container_name: oai-ext-dn
         image: oaisoftwarealliance/trf-gen-cn5g:latest
-        environment:
-            - UPF_FQDN=edgecomllc-eupf
-            - UE_NETWORK=12.1.1.0/24
-            - USE_FQDN=yes
+        entrypoint: /bin/bash -c \
+              "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\
+              "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity"
+        command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"]
         healthcheck:
             test: /bin/bash -c "ip r | grep 12.1.1"
             interval: 10s

docker-compose/docker-compose-pfcp-oai-upf-dev.yaml

@@ -5,22 +5,24 @@ services:
         image: tariromukute/pfcp-kitchen-sink:latest
         volumes:
             - ./sessions-oai-upf.yaml:/app/sessions.yaml
-        # command: ./pfcpclient -r 192.168.70.134:8805 -s sessions.yaml
-        command: tail -f /dev/null
+        command: ./pfcpclient -r 192.168.70.134:8805 -s sessions.yaml
+        # command: tail -f /dev/null
         depends_on:
-            - oai-upf
+            # - oai-upf
+            - edgecomllc-eupf
         networks:
             public_net:
                 ipv4_address: 192.168.70.131
     tc-gtpu-tnl:
         privileged: true
+        # platform: linux/amd64
         container_name: "tc-gtpu-tnl"
-        image: tariromukute/tc-gtpu:latest
+        image: tariromukute/tc-gtpu-tnl:latest
         command: tail -f /dev/null
         volumes:
             - /sys/kernel/debug/:/sys/kernel/debug/
-            - /sys/fs/bpf:/sys/fs/bpf 
-            - /home/ubuntu/tc-gtpu-tnl:/home
+            - /sys/fs/bpf:/sys/fs/bpf
+            - /Users/tariromukute/Documents/phd/tc-gtpu:/home
         devices:
             - /dev/net/tun:/dev/net/tun
         # cap_add:
@@ -29,34 +31,76 @@ services:
         # cap_drop:
         #     - ALL
         depends_on:
-            - oai-upf
+            # - oai-upf
+            - edgecomllc-eupf
             - pfcp-kitchen-sink
         networks:
+            n6_net:
+                ipv4_address: 192.168.72.130
             public_net:
                 ipv4_address: 192.168.70.130
-    oai-upf:
-        container_name: "oai-upf"
-        image: tariromukute/oai-upf-dev:latest
-        command: tail -f /dev/null
-        volumes:
-            - ./basic_config.yaml:/openair-upf/etc/config.yaml
-            - /home/ubuntu/oai-cn5g-upf:/home
+    # oai-upf:
+    #     container_name: "oai-upf"
+    #     image: tariromukute/oai-upf-dev:latest
+    #     command: tail -f /dev/null
+    #     volumes:
+    #         - ./basic_config.yaml:/openair-upf/etc/config.yaml
+    #         - /home/ubuntu/oai-cn5g-upf:/home
+    #     environment:
+    #         - TZ=Europe/Paris
+    #         - N6_MAC=02:42:ac:11:65:43
+    #         - LAN_MAC=02:42:ac:11:65:44
+    #     cap_add:
+    #         - NET_ADMIN
+    #         - SYS_ADMIN
+    #     cap_drop:
+    #         - ALL
+    #     privileged: true
+    #     networks:
+    #         public_net:
+    #             ipv4_address: 192.168.70.134
+    #         n6_net:
+    #             ipv4_address: 192.168.72.134
+    #             mac_address: 02:42:ac:11:65:43
+    edgecomllc-eupf:
+        platform: linux/amd64
+        container_name: "edgecomllc-eupf"
+        image: ghcr.io/edgecomllc/eupf:main
+        entrypoint:
+        - /bin/sh
+        - -c
+        - |
+            ip route del default;
+            ip route add default via 192.168.72.135 dev eth0 &&
+            sh /app/bin/entrypoint.sh
         environment:
-            - TZ=Europe/Paris
-            - N6_MAC=02:42:ac:11:65:43
-            - LAN_MAC=02:42:ac:11:65:44
+            - UPF_INTERFACE_NAME=eth1,eth0
+            - UPF_XDP_ATTACH_MODE=generic
+            - UPF_API_ADDRESS=:8080
+            - UPF_PFCP_ADDRESS=:8805
+            - UPF_METRICS_ADDRESS=:9091
+            - UPF_PFCP_NODE_ID=192.168.70.134
+            - UPF_N3_ADDRESS=192.168.70.134
+            - UPF_UEIP_POOL=12.1.1.0/24
+            - UPF_LOGGING_LEVEL=debug
         cap_add:
             - NET_ADMIN
             - SYS_ADMIN
-        cap_drop:
-            - ALL
+            - SYS_RESOURCE # setrlimit
+        # cap_drop:
+        #     - ALL
+        ports:
+            - "127.0.0.1:8081:8081"
+            - "127.0.0.1:8880:8080"
+            - "127.0.0.1:9090:9090"
+        sysctls:
+            - net.ipv4.conf.all.forwarding=1
         privileged: true
         networks:
             public_net:
                 ipv4_address: 192.168.70.134
             n6_net:
                 ipv4_address: 192.168.72.134
-                mac_address: 02:42:ac:11:65:43
     oai-ext-dn:
         privileged: true
         init: true
@@ -67,8 +111,6 @@ services:
               "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\
               "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity"
         command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"]
-        volumes:
-            - /home/ubuntu/trf-gen-cn5g:/home
         healthcheck:
             test: /bin/bash -c "iptables -L -t nat | grep MASQUERADE"
             interval: 10s
@@ -84,15 +126,9 @@ services:
             #     ipv4_address: 192.168.73.135
             #     mac_address: 02:42:ac:11:65:45
 
-    iperf:
-        container_name: "iperf"
-        image: cagedata/iperf3:latest
-        networks:
-            n6_net:
-                ipv4_address: 192.168.72.136
-
     iperf3:
         privileged: true
+        platform: linux/amd64
         container_name: "iperf3"
         image: ubuntu:jammy
         command: tail -f /dev/null
@@ -104,6 +140,7 @@ services:
 
     dropwatch:
         privileged: true
+        platform: linux/amd64
         container_name: "dropwatch"
         image: ubuntu:jammy
         command: tail -f /dev/null

docker-compose/sessions-eupf.yaml

@@ -4,7 +4,7 @@
     precedence: 0
     pdi:
       sourceInterface: Access
-      networkInstance: access.oai.org
+      # networkInstance: access.oai.org
       localFTEID:
         teid: 1234
         ip4: 192.168.70.134
@@ -18,14 +18,14 @@
     applyAction: Forward
     forwardingParameters:
       destinationInterface: Core
-      networkInstance: internet.oai.org.pri
+      # networkInstance: internet.oai.org.pri
 - seid: 1
   pdrs:
   - pdrID: 1
     precedence: 0
     pdi:
       sourceInterface: Core
-      networkInstance: internet.oai.org.pri
+      # networkInstance: internet.oai.org.pri
       ueIPAddress:
         isDestination: true
         ip4: 12.1.1.2
@@ -35,7 +35,7 @@
     applyAction: Forward
     forwardingParameters:
       destinationInterface: Access
-      networkInstance: access.oai.org
+      # networkInstance: access.oai.org
       outerHeaderCreation:
         desc: OUTER_HEADER_CREATION_GTPU_UDP_IPV4
         teid: 1234

src/tc-gtpu.bpf.c

@@ -104,14 +104,12 @@ static struct ipv6_gtpu_encap ipv6_gtpu_encap = {
 };
 
 /* Logic for checksum, thanks to https://github.com/facebookincubator/katran/blob/main/katran/lib/bpf/csum_helpers.h */
-static __always_inline __u16 csum_fold_helper(__u64 csum) {
-    int i;
-#pragma unroll
-    for (i = 0; i < 4; i++) {
-        if (csum >> 16)
-            csum = (csum & 0xffff) + (csum >> 16);
-    }
-    return ~csum;
+static __always_inline __u16 csum_fold_helper(__u32 csum)
+{
+	__u32 sum;
+	sum = (csum >> 16) + (csum & 0xffff);
+	sum += (sum >> 16);
+	return ~sum;
 }
 
 static __always_inline void ipv4_csum(void* data_start, int data_size, __u64* csum) {
@@ -270,6 +268,7 @@ int tnl_if_ingress_fn(struct __sk_buff *skb)
     __u32 qfi, teid;
     __u32 key = skb->ifindex;
     struct egress_state *state;
+    __u64 flags;
 
     int payload_len = (data_end - data) - sizeof(struct ethhdr);
 
@@ -281,7 +280,21 @@ int tnl_if_ingress_fn(struct __sk_buff *skb)
         bpf_printk("No match going out");
         goto out;
     }
-
+
+    ip_type = parse_iphdr(&nh, data_end, &iphdr);
+	if (ip_type < 0)
+		goto out;
+
+    // check if skb is non-linear, it if is and pull in non-linear data
+    if (bpf_ntohs(iphdr->tot_len) > payload_len)
+        if (bpf_skb_pull_data(skb, bpf_ntohs(iphdr->tot_len) + sizeof(struct ethhdr)) < 0)
+            return TC_ACT_UNSPEC;
+
+    data_end = (void *)(unsigned long long)skb->data_end;
+    data = (void *)(unsigned long long)skb->data;
+    nh.pos = data;
+    payload_len = (data_end - data) - sizeof(struct ethhdr);
+
     // Logic to fetch QFI and TEID from maps or use defaults
     state = bpf_map_lookup_elem(&egress_map, &key);
     if (state && state->teid && state->qfi) {
@@ -291,9 +304,13 @@ int tnl_if_ingress_fn(struct __sk_buff *skb)
         qfi = DEFAULT_QFI;
         teid = skb->ifindex;  // Use interface index as default TEID
     }
-
+
+    flags = // BPF_F_ADJ_ROOM_FIXED_GSO |
+          BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 |
+          BPF_F_ADJ_ROOM_ENCAP_L4_UDP;
+
     int roomlen = sizeof(struct ipv4_gtpu_encap);
-    int ret = bpf_skb_adjust_room(skb, roomlen, BPF_ADJ_ROOM_MAC, 0);
+    int ret = bpf_skb_adjust_room(skb, roomlen, BPF_ADJ_ROOM_MAC, flags);
     if (ret) {
         bpf_printk("error calling skb adjust room %d, error code %d\n", roomlen, ret);
         return TC_ACT_SHOT;
@@ -307,6 +324,10 @@ int tnl_if_ingress_fn(struct __sk_buff *skb)
     ipv4_gtpu_encap.ipv4h.daddr = config.daddr.addr.addr4.s_addr;
     ipv4_gtpu_encap.ipv4h.saddr = config.saddr.addr.addr4.s_addr;
     ipv4_gtpu_encap.ipv4h.tot_len = bpf_htons(sizeof(struct ipv4_gtpu_encap) + payload_len);
+
+    // For checksum to be recalculated
+    bpf_set_hash_invalid(skb);
+    // ipv4_gtpu_encap.ipv4h.check = csum_fold_helper(bpf_csum_diff((__be32 *)&ipv4_gtpu_encap.ipv4h, 0, (__be32 *)&ipv4_gtpu_encap.ipv4h, sizeof(struct iphdr), 0));
 
     ipv4_gtpu_encap.udp.len = bpf_htons(sizeof(struct ipv4_gtpu_encap) + payload_len - sizeof(struct iphdr));