From 382913950669fc9cd22c54c23bac597f08128303 Mon Sep 17 00:00:00 2001 From: Tariro Mukute Date: Tue, 9 Apr 2024 16:03:11 +0200 Subject: [PATCH] Fix TCP session establishment not completing * Test with eUPF --- .DS_Store | Bin 6148 -> 8196 bytes .dockerignore | 3 +- .gitignore | 1 + .vscode/settings.json | 3 +- NOTES.md | 15 ++- docker-compose/.DS_Store | Bin 0 -> 8196 bytes docker-compose/docker-compose-pfcp-eupf.yaml | 27 +++-- .../docker-compose-pfcp-oai-upf-dev.yaml | 95 ++++++++++++------ docker-compose/sessions-eupf.yaml | 8 +- src/tc-gtpu.bpf.c | 43 ++++++-- 10 files changed, 141 insertions(+), 54 deletions(-) create mode 100644 docker-compose/.DS_Store diff --git a/.DS_Store b/.DS_Store index 8ff218e631e44b5eb2ea83ec17c36cc0761e2aa0..0393afe45c6242cb8e7e797c13825fa9f377aaed 100644 GIT binary patch literal 8196 zcmeHML2lDP6n&FKG=YLDA)#&%zCa=sIDlKyMHdB$Z~z*o4H0e}r6hu#_uPRiAP&P} zx`Oxr43X{FmDof`{E24X#Qtyo%$pfcz5~FWtR_dmA%HHk;K2hHHA(JE-b$@-1pI$ZGVLWJVf zJ9&|s!^Ix0GzCn7Z3QHD-@_OST;WqOe$U3d!_DNr5(mFDDg6;9v`g?2msq0DT83-# zGu+_P$Nn4c?cZ=GH%U6qvQ&2Xole%H$H{CI zt-72;X}h6S-2z4XUa&uEWXQs=V?1wS=&aWBP;)1n? z@$333^7gQn9i{7ih0kSmb#-m5xj|$0Q#NYuQMRmn_XoJ0|0;(Srohe>XnT)`691nz zzyIHPzs#X2U<&L`0oNIhh9~srkLGR1a#Tmm@0lfuUG7mSEc7%dYB)~Z{9(v?L{s6! U0dtSM!el=LR0b#*NOcoF_nA{*_w6Sa%%VKs84nbz1J|Ga_1`@6y^EMWKXP(S2 X;|Q`4WFiZQW`fuV7TX-pGlv-fQ=J$| diff --git a/.dockerignore b/.dockerignore index ddd2aad..95d12d7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,5 @@ docs/ *.o *.pcap -*.log \ No newline at end of file +*.log +*.pcapng \ No newline at end of file diff --git a/.gitignore b/.gitignore index 42918ea..661e2b4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ *.ll *.pcap +*.pcapng *.log \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 328a896..1d36155 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,7 @@ "unistd.h": "c", "signal.h": "c", "gtpu.h": "c", - "tc-gtpu.skel.h": "c" + "tc-gtpu.skel.h": "c", + "ios": "c" } } \ No newline at end of file diff --git a/NOTES.md b/NOTES.md index 66b70e4..246779e 100644 --- a/NOTES.md +++ b/NOTES.md @@ -28,6 +28,10 @@ cat /sys/kernel/debug/tracing/trace_pipe Build with Dockerfile +```bash +docker buildx build --platform=linux/amd64 -t tariromukute/tc-gtpu:latest -f Dockerfile . +``` + Either: 1. create container and the mount the debugfs inside the container. @@ -184,6 +188,13 @@ ethtool --offload rx off tx off # Check for TCP related errors netstat -s +``` + +```bash +sysctl net.ipv4.tcp_timestamps + +sysctl -w net.ipv4.tcp_timestamps=0 + ``` ## Useful Resources @@ -196,4 +207,6 @@ netstat -s - https://lore.kernel.org/bpf/1567892444-16344-2-git-send-email-alan.maguire@oracle.com/ - https://github.com/siemens/edgeshark?tab=readme-ov-file#siemens-industrial-edge - https://www.dasblinkenlichten.com/working-with-tc-on-linux-systems/ -- https://www.alibabacloud.com/blog/why-are-linux-kernel-protocol-stacks-dropping-syn-packets_595251 \ No newline at end of file +- https://www.alibabacloud.com/blog/why-are-linux-kernel-protocol-stacks-dropping-syn-packets_595251 +- https://arstechnica.com/civis/threads/a-possibly-simple-sniffer-trace-question-psh-ack.343792/ +- https://blogs.oracle.com/linux/post/notes-on-bpf-7-bpf-tc-and-generic-segmentation-offload \ No newline at end of file diff --git a/docker-compose/.DS_Store b/docker-compose/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..44f00d19c5d3c61c2ac220b0c50ff5890eab608a GIT binary patch literal 8196 zcmeHM%T60H6g^I|Kz*V zbk&}FZP75nOvwT&^;+^h6Z<|q$9A0S0C0y_qkUitz#6OIs}~ga6zP|`RC=-HCDG9y zC~(T^F~#R4ZEG+Fi~(c77%&Emfq#Jkp4n2?66b#CHE9eO1OFuh^8Jvp3g!`;fcn)z zp(y}yz-BFcW*(q6S;RbI6Hra*Gh`1gn!1=`xM0ryOw@^a#3o=chYRL#F|vypiqq({ z%TygM4`|XDFb0+xklcM8LwrJw$({Utf-&~+l|2`zK32XMR(?or*y3zS7M%ggP*N9?^lk~F$=wME7Iz3G$(VHsWj&XKsz>C$nkgr$UmnZ|G92b<4zwr6s^ytQ3^tjpSF)IZ@Mb;_V-Xe9ovUHhIJ*AX`}Mhk(*x!Wei`27UwkXdAcy literal 0 HcmV?d00001 diff --git a/docker-compose/docker-compose-pfcp-eupf.yaml b/docker-compose/docker-compose-pfcp-eupf.yaml index 8dee907..303f9e3 100644 --- a/docker-compose/docker-compose-pfcp-eupf.yaml +++ b/docker-compose/docker-compose-pfcp-eupf.yaml @@ -13,6 +13,7 @@ services: ipv4_address: 192.168.70.131 tc-gtpu-tnl: privileged: true + platform: linux/amd64 container_name: "tc-gtpu-tnl" image: tariromukute/tc-gtpu:latest command: tail -f /dev/null @@ -32,16 +33,26 @@ services: public_net: ipv4_address: 192.168.70.130 edgecomllc-eupf: + platform: linux/amd64 container_name: "edgecomllc-eupf" image: ghcr.io/edgecomllc/eupf:main + entrypoint: + - /bin/sh + - -c + - | + ip route del default; + ip route add default via 192.168.72.135 dev eth0 && + sh /app/bin/entrypoint.sh environment: - UPF_INTERFACE_NAME=eth1,eth0 - UPF_XDP_ATTACH_MODE=generic - - UPF_API_ADDRESS=:8081 + - UPF_API_ADDRESS=:8080 - UPF_PFCP_ADDRESS=:8805 - UPF_METRICS_ADDRESS=:9091 - UPF_PFCP_NODE_ID=192.168.70.134 - UPF_N3_ADDRESS=192.168.70.134 + - UPF_UEIP_POOL=12.1.1.0/24 + - UPF_LOGGING_LEVEL=debug cap_add: - NET_ADMIN - SYS_ADMIN @@ -49,8 +60,9 @@ services: # cap_drop: # - ALL ports: - - "8880:8080" - - "9090:9090" + - "127.0.0.1:8081:8081" + - "127.0.0.1:8880:8080" + - "127.0.0.1:9090:9090" sysctls: - net.ipv4.conf.all.forwarding=1 privileged: true @@ -61,13 +73,14 @@ services: ipv4_address: 192.168.72.134 oai-ext-dn: privileged: true + platform: linux/amd64 init: true container_name: oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:latest - environment: - - UPF_FQDN=edgecomllc-eupf - - UE_NETWORK=12.1.1.0/24 - - USE_FQDN=yes + entrypoint: /bin/bash -c \ + "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ + "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity" + command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"] healthcheck: test: /bin/bash -c "ip r | grep 12.1.1" interval: 10s diff --git a/docker-compose/docker-compose-pfcp-oai-upf-dev.yaml b/docker-compose/docker-compose-pfcp-oai-upf-dev.yaml index b20e2c6..46df4d2 100644 --- a/docker-compose/docker-compose-pfcp-oai-upf-dev.yaml +++ b/docker-compose/docker-compose-pfcp-oai-upf-dev.yaml @@ -5,22 +5,24 @@ services: image: tariromukute/pfcp-kitchen-sink:latest volumes: - ./sessions-oai-upf.yaml:/app/sessions.yaml - # command: ./pfcpclient -r 192.168.70.134:8805 -s sessions.yaml - command: tail -f /dev/null + command: ./pfcpclient -r 192.168.70.134:8805 -s sessions.yaml + # command: tail -f /dev/null depends_on: - - oai-upf + # - oai-upf + - edgecomllc-eupf networks: public_net: ipv4_address: 192.168.70.131 tc-gtpu-tnl: privileged: true + # platform: linux/amd64 container_name: "tc-gtpu-tnl" - image: tariromukute/tc-gtpu:latest + image: tariromukute/tc-gtpu-tnl:latest command: tail -f /dev/null volumes: - /sys/kernel/debug/:/sys/kernel/debug/ - - /sys/fs/bpf:/sys/fs/bpf - - /home/ubuntu/tc-gtpu-tnl:/home + - /sys/fs/bpf:/sys/fs/bpf + - /Users/tariromukute/Documents/phd/tc-gtpu:/home devices: - /dev/net/tun:/dev/net/tun # cap_add: @@ -29,34 +31,76 @@ services: # cap_drop: # - ALL depends_on: - - oai-upf + # - oai-upf + - edgecomllc-eupf - pfcp-kitchen-sink networks: + n6_net: + ipv4_address: 192.168.72.130 public_net: ipv4_address: 192.168.70.130 - oai-upf: - container_name: "oai-upf" - image: tariromukute/oai-upf-dev:latest - command: tail -f /dev/null - volumes: - - ./basic_config.yaml:/openair-upf/etc/config.yaml - - /home/ubuntu/oai-cn5g-upf:/home + # oai-upf: + # container_name: "oai-upf" + # image: tariromukute/oai-upf-dev:latest + # command: tail -f /dev/null + # volumes: + # - ./basic_config.yaml:/openair-upf/etc/config.yaml + # - /home/ubuntu/oai-cn5g-upf:/home + # environment: + # - TZ=Europe/Paris + # - N6_MAC=02:42:ac:11:65:43 + # - LAN_MAC=02:42:ac:11:65:44 + # cap_add: + # - NET_ADMIN + # - SYS_ADMIN + # cap_drop: + # - ALL + # privileged: true + # networks: + # public_net: + # ipv4_address: 192.168.70.134 + # n6_net: + # ipv4_address: 192.168.72.134 + # mac_address: 02:42:ac:11:65:43 + edgecomllc-eupf: + platform: linux/amd64 + container_name: "edgecomllc-eupf" + image: ghcr.io/edgecomllc/eupf:main + entrypoint: + - /bin/sh + - -c + - | + ip route del default; + ip route add default via 192.168.72.135 dev eth0 && + sh /app/bin/entrypoint.sh environment: - - TZ=Europe/Paris - - N6_MAC=02:42:ac:11:65:43 - - LAN_MAC=02:42:ac:11:65:44 + - UPF_INTERFACE_NAME=eth1,eth0 + - UPF_XDP_ATTACH_MODE=generic + - UPF_API_ADDRESS=:8080 + - UPF_PFCP_ADDRESS=:8805 + - UPF_METRICS_ADDRESS=:9091 + - UPF_PFCP_NODE_ID=192.168.70.134 + - UPF_N3_ADDRESS=192.168.70.134 + - UPF_UEIP_POOL=12.1.1.0/24 + - UPF_LOGGING_LEVEL=debug cap_add: - NET_ADMIN - SYS_ADMIN - cap_drop: - - ALL + - SYS_RESOURCE # setrlimit + # cap_drop: + # - ALL + ports: + - "127.0.0.1:8081:8081" + - "127.0.0.1:8880:8080" + - "127.0.0.1:9090:9090" + sysctls: + - net.ipv4.conf.all.forwarding=1 privileged: true networks: public_net: ipv4_address: 192.168.70.134 n6_net: ipv4_address: 192.168.72.134 - mac_address: 02:42:ac:11:65:43 oai-ext-dn: privileged: true init: true @@ -67,8 +111,6 @@ services: "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity" command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"] - volumes: - - /home/ubuntu/trf-gen-cn5g:/home healthcheck: test: /bin/bash -c "iptables -L -t nat | grep MASQUERADE" interval: 10s @@ -84,15 +126,9 @@ services: # ipv4_address: 192.168.73.135 # mac_address: 02:42:ac:11:65:45 - iperf: - container_name: "iperf" - image: cagedata/iperf3:latest - networks: - n6_net: - ipv4_address: 192.168.72.136 - iperf3: privileged: true + platform: linux/amd64 container_name: "iperf3" image: ubuntu:jammy command: tail -f /dev/null @@ -104,6 +140,7 @@ services: dropwatch: privileged: true + platform: linux/amd64 container_name: "dropwatch" image: ubuntu:jammy command: tail -f /dev/null diff --git a/docker-compose/sessions-eupf.yaml b/docker-compose/sessions-eupf.yaml index 236e756..a62d468 100644 --- a/docker-compose/sessions-eupf.yaml +++ b/docker-compose/sessions-eupf.yaml @@ -4,7 +4,7 @@ precedence: 0 pdi: sourceInterface: Access - networkInstance: access.oai.org + # networkInstance: access.oai.org localFTEID: teid: 1234 ip4: 192.168.70.134 @@ -18,14 +18,14 @@ applyAction: Forward forwardingParameters: destinationInterface: Core - networkInstance: internet.oai.org.pri + # networkInstance: internet.oai.org.pri - seid: 1 pdrs: - pdrID: 1 precedence: 0 pdi: sourceInterface: Core - networkInstance: internet.oai.org.pri + # networkInstance: internet.oai.org.pri ueIPAddress: isDestination: true ip4: 12.1.1.2 @@ -35,7 +35,7 @@ applyAction: Forward forwardingParameters: destinationInterface: Access - networkInstance: access.oai.org + # networkInstance: access.oai.org outerHeaderCreation: desc: OUTER_HEADER_CREATION_GTPU_UDP_IPV4 teid: 1234 diff --git a/src/tc-gtpu.bpf.c b/src/tc-gtpu.bpf.c index f1f8858..74b7286 100644 --- a/src/tc-gtpu.bpf.c +++ b/src/tc-gtpu.bpf.c @@ -104,14 +104,12 @@ static struct ipv6_gtpu_encap ipv6_gtpu_encap = { }; /* Logic for checksum, thanks to https://github.com/facebookincubator/katran/blob/main/katran/lib/bpf/csum_helpers.h */ -static __always_inline __u16 csum_fold_helper(__u64 csum) { - int i; -#pragma unroll - for (i = 0; i < 4; i++) { - if (csum >> 16) - csum = (csum & 0xffff) + (csum >> 16); - } - return ~csum; +static __always_inline __u16 csum_fold_helper(__u32 csum) +{ + __u32 sum; + sum = (csum >> 16) + (csum & 0xffff); + sum += (sum >> 16); + return ~sum; } static __always_inline void ipv4_csum(void* data_start, int data_size, __u64* csum) { @@ -270,6 +268,7 @@ int tnl_if_ingress_fn(struct __sk_buff *skb) __u32 qfi, teid; __u32 key = skb->ifindex; struct egress_state *state; + __u64 flags; int payload_len = (data_end - data) - sizeof(struct ethhdr); @@ -281,7 +280,21 @@ int tnl_if_ingress_fn(struct __sk_buff *skb) bpf_printk("No match going out"); goto out; } - + + ip_type = parse_iphdr(&nh, data_end, &iphdr); + if (ip_type < 0) + goto out; + + // check if skb is non-linear, it if is and pull in non-linear data + if (bpf_ntohs(iphdr->tot_len) > payload_len) + if (bpf_skb_pull_data(skb, bpf_ntohs(iphdr->tot_len) + sizeof(struct ethhdr)) < 0) + return TC_ACT_UNSPEC; + + data_end = (void *)(unsigned long long)skb->data_end; + data = (void *)(unsigned long long)skb->data; + nh.pos = data; + payload_len = (data_end - data) - sizeof(struct ethhdr); + // Logic to fetch QFI and TEID from maps or use defaults state = bpf_map_lookup_elem(&egress_map, &key); if (state && state->teid && state->qfi) { @@ -291,9 +304,13 @@ int tnl_if_ingress_fn(struct __sk_buff *skb) qfi = DEFAULT_QFI; teid = skb->ifindex; // Use interface index as default TEID } - + + flags = // BPF_F_ADJ_ROOM_FIXED_GSO | + BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | + BPF_F_ADJ_ROOM_ENCAP_L4_UDP; + int roomlen = sizeof(struct ipv4_gtpu_encap); - int ret = bpf_skb_adjust_room(skb, roomlen, BPF_ADJ_ROOM_MAC, 0); + int ret = bpf_skb_adjust_room(skb, roomlen, BPF_ADJ_ROOM_MAC, flags); if (ret) { bpf_printk("error calling skb adjust room %d, error code %d\n", roomlen, ret); return TC_ACT_SHOT; @@ -307,6 +324,10 @@ int tnl_if_ingress_fn(struct __sk_buff *skb) ipv4_gtpu_encap.ipv4h.daddr = config.daddr.addr.addr4.s_addr; ipv4_gtpu_encap.ipv4h.saddr = config.saddr.addr.addr4.s_addr; ipv4_gtpu_encap.ipv4h.tot_len = bpf_htons(sizeof(struct ipv4_gtpu_encap) + payload_len); + + // For checksum to be recalculated + bpf_set_hash_invalid(skb); + // ipv4_gtpu_encap.ipv4h.check = csum_fold_helper(bpf_csum_diff((__be32 *)&ipv4_gtpu_encap.ipv4h, 0, (__be32 *)&ipv4_gtpu_encap.ipv4h, sizeof(struct iphdr), 0)); ipv4_gtpu_encap.udp.len = bpf_htons(sizeof(struct ipv4_gtpu_encap) + payload_len - sizeof(struct iphdr));