Run eUPF with ovs based n6-lan

* Chores and clean up of logging, namings and comments
tariromukute · Apr 18, 2024 · ece0765 · ece0765
1 parent 3829139
commit ece0765
Show file tree

Hide file tree

Showing 9 changed files with 257 additions and 195 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/Dockerfile b/Dockerfile
@@ -21,6 +21,7 @@ FROM $BASE_IMAGE AS runtime
 
 RUN apt-get update && \
     apt-get install -y iproute2 iputils-ping tcpdump \
+    iperf3 \
     libelf1 libelf-dev zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
 
@@ -35,6 +36,6 @@ RUN apt-get autoremove -y && \
 WORKDIR /app
 
 COPY --from=builder /app/src /app/
-COPY ./entrypoint.sh /app/bin/entrypoint.sh
+COPY ./entrypoint.sh /app/entrypoint.sh
 
-# ENTRYPOINT ./entrypoint.sh
+ENTRYPOINT [ "sh", "/app/entrypoint.sh" ]
diff --git a/NOTES.md b/NOTES.md
@@ -103,7 +103,7 @@ tcpdump -i eth0 -w tmp.pcap
 Or
 
 ```bash
-./tc-gtpu -g eth0 -i uegtp -s 192.168.70.130 -d 192.168.70.134 -u 12.1.1.2 -b 12.1.1.1 --ul-teid 1234 --dl-teid 1234 -q 9 -n 2 -f /home/tu-gtpu.pcap -vvv
+./tc-gtpu -g eth0 -i uegtp -s 192.168.71.130 -d 192.168.71.134 -u 12.1.1.2 -b 12.1.1.1 --ul-teid 1234 --dl-teid 1234 -q 9 -n 2 -f /home/tu-gtpu.pcap -vvv
 ```
 
 ```bash
@@ -196,6 +196,7 @@ sysctl net.ipv4.tcp_timestamps
 sysctl -w net.ipv4.tcp_timestamps=0
 
 ```
+
 ## Useful Resources
 
 - [Understanding tc “direct action” mode for BPF](https://qmonnet.github.io/whirl-offload/2020/04/11/tc-bpf-direct-action/)
@@ -209,4 +210,5 @@ sysctl -w net.ipv4.tcp_timestamps=0
 - https://www.dasblinkenlichten.com/working-with-tc-on-linux-systems/
 - https://www.alibabacloud.com/blog/why-are-linux-kernel-protocol-stacks-dropping-syn-packets_595251
 - https://arstechnica.com/civis/threads/a-possibly-simple-sniffer-trace-question-psh-ack.343792/
-- https://blogs.oracle.com/linux/post/notes-on-bpf-7-bpf-tc-and-generic-segmentation-offload
+- https://blogs.oracle.com/linux/post/notes-on-bpf-7-bpf-tc-and-generic-segmentation-offload
+- https://github.com/antrea-io/antrea/issues/593
diff --git a/docker-compose/docker-compose-pfcp-eupf.yaml b/docker-compose/docker-compose-pfcp-eupf.yaml
@@ -9,29 +9,36 @@ services:
         depends_on:
             - edgecomllc-eupf
         networks:
-            public_net:
+            n4_net:
                 ipv4_address: 192.168.70.131
-    tc-gtpu-tnl:
-        privileged: true
-        platform: linux/amd64
-        container_name: "tc-gtpu-tnl"
+    ue-sim:
+        privileged: true # So it can create UE namespaces
+        container_name: "ue-sim"
         image: tariromukute/tc-gtpu:latest
-        command: tail -f /dev/null
+        command:
+        - /bin/bash
+        - -c
+        - |
+            ./tc-gtpu -g eth0 -i uegtp -s 192.168.71.130 -d 192.168.71.134 \
+            -u 12.1.1.2 -b 12.1.1.1 --ul-teid 1234 --dl-teid 1234 --qfi 9 \
+            -n 2 -f /home/tu-gtpu.pcap -vvv
+        healthcheck:
+            test: ip netns exec uegtp0 ping -c 4 192.168.73.129 || exit 1
+            interval: 10s
+            timeout: 5s
+            retries: 5
         volumes:
             - /sys/kernel/debug/:/sys/kernel/debug/
+            - /sys/fs/bpf:/sys/fs/bpf
         devices:
             - /dev/net/tun:/dev/net/tun
-        cap_add:
-            - NET_ADMIN
-            - SYS_ADMIN
-        # cap_drop:
-        #     - ALL
         depends_on:
             - edgecomllc-eupf
             - pfcp-kitchen-sink
         networks:
-            public_net:
-                ipv4_address: 192.168.70.130
+            n3_net:
+                ipv4_address: 192.168.71.130
+
     edgecomllc-eupf:
         platform: linux/amd64
         container_name: "edgecomllc-eupf"
@@ -41,24 +48,24 @@ services:
         - -c
         - |
             ip route del default;
-            ip route add default via 192.168.72.135 dev eth0 &&
+            ip route add default via 192.168.72.138 dev eth2 &&
             sh /app/bin/entrypoint.sh
         environment:
-            - UPF_INTERFACE_NAME=eth1,eth0
+            - UPF_INTERFACE_NAME=eth0,eth2
             - UPF_XDP_ATTACH_MODE=generic
             - UPF_API_ADDRESS=:8080
             - UPF_PFCP_ADDRESS=:8805
             - UPF_METRICS_ADDRESS=:9091
             - UPF_PFCP_NODE_ID=192.168.70.134
-            - UPF_N3_ADDRESS=192.168.70.134
+            - UPF_N3_ADDRESS=192.168.71.134
             - UPF_UEIP_POOL=12.1.1.0/24
             - UPF_LOGGING_LEVEL=debug
         cap_add:
             - NET_ADMIN
             - SYS_ADMIN
             - SYS_RESOURCE # setrlimit
-        # cap_drop:
-        #     - ALL
+        cap_drop:
+            - ALL
         ports:
             - "127.0.0.1:8081:8081"
             - "127.0.0.1:8880:8080"
@@ -67,52 +74,86 @@ services:
             - net.ipv4.conf.all.forwarding=1
         privileged: true
         networks:
-            public_net:
+            n4_net:
                 ipv4_address: 192.168.70.134
+            n3_net:
+                ipv4_address: 192.168.71.134
             n6_net:
                 ipv4_address: 192.168.72.134
-    oai-ext-dn:
-        privileged: true
+    n6-lan:
         platform: linux/amd64
+        privileged: true
         init: true
-        container_name: oai-ext-dn
-        image: oaisoftwarealliance/trf-gen-cn5g:latest
-        entrypoint: /bin/bash -c \
-              "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\
-              "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity"
-        command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"]
-        healthcheck:
-            test: /bin/bash -c "ip r | grep 12.1.1"
-            interval: 10s
-            timeout: 5s
-            retries: 5
+        container_name: "n6-lan"
+        image: tariromukute/n6-lan-simple:latest
+        command: 
+        - /bin/bash
+        - -c
+        - |
+            sh testovs.sh
+            ovs-vsctl add-port brovs1 eth1
+            ip addr flush dev eth1 && ip addr add 192.168.72.138/26 dev brovs1 && ip link set brovs1 up
+            iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+            ip route add 12.1.1.0/24 via 192.168.72.134 dev brovs1
+            tail -f /dev/null
+        devices:
+            - /dev/net/tun:/dev/net/tun # https://docs.openvswitch.org/en/stable/intro/install/userspace/#building-and-installing
+        volumes:
+            - /lib/modules:/lib/modules
         networks:
-            public_net:
-                ipv4_address: 192.168.70.135
             n6_net:
-                ipv4_address: 192.168.72.135
+                ipv4_address: 192.168.72.138
+            data_net:
+                ipv4_address: 192.168.73.138
+
+    iperf3:
+        privileged: true
+        platform: linux/amd64
+        container_name: "iperf3"
+        image: ubuntu:jammy
+        command: 
+        - /bin/bash
+        - -c
+        - |
+            apt update -y
+            apt install iperf3 -y
+            iperf3 -s
+        cap_add:
+            - NET_ADMIN
+        networks:
+            data_net:
+                ipv4_address: 192.168.73.137
+
 networks:
-    public_net:
+    n4_net:
         driver: bridge
-        name: demo-oai-public-net
+        name: demo-n4-net
         ipam:
             config:
                 - subnet: 192.168.70.128/26
         driver_opts:
-            com.docker.network.bridge.name: "demo-oai"
+            com.docker.network.bridge.name: "demo-n4"
     n3_net:
-        name: demo-oai-n3-net
+        driver: bridge
+        name: demo-n3-net
         ipam:
             config:
                 - subnet: 192.168.71.128/26
         driver_opts:
             com.docker.network.bridge.name: "demo-n3"
     n6_net:
-        name: demo-oai-n6-net
+        driver: bridge
+        name: demo-n6-net
         ipam:
-            options:
-                iface: n6
             config:
                 - subnet: 192.168.72.128/26
         driver_opts:
-            com.docker.network.bridge.name: "demo-n6"
+            com.docker.network.bridge.name: "demo-n6"
+    data_net:
+        driver: bridge
+        name: demo-data-net
+        ipam:
+            config:
+                - subnet: 192.168.73.128/26
+        driver_opts:
+            com.docker.network.bridge.name: "demo-dn"
diff --git a/docker-compose/docker-compose-pfcp-oai-upf-dev.yaml b/docker-compose/docker-compose-pfcp-oai-upf-dev.yaml
@@ -17,7 +17,7 @@ services:
         privileged: true
         # platform: linux/amd64
         container_name: "tc-gtpu-tnl"
-        image: tariromukute/tc-gtpu-tnl:latest
+        image: tariromukute/tc-gtpu:latest
         command: tail -f /dev/null
         volumes:
             - /sys/kernel/debug/:/sys/kernel/debug/
@@ -71,7 +71,7 @@ services:
         - -c
         - |
             ip route del default;
-            ip route add default via 192.168.72.135 dev eth0 &&
+            ip route add default via 192.168.72.138 dev eth0 &&
             sh /app/bin/entrypoint.sh
         environment:
             - UPF_INTERFACE_NAME=eth1,eth0
@@ -101,55 +101,96 @@ services:
                 ipv4_address: 192.168.70.134
             n6_net:
                 ipv4_address: 192.168.72.134
-    oai-ext-dn:
+    n6-lan:
+        platform: linux/amd64
         privileged: true
         init: true
-        container_name: "oai-ext-dn"
-        image: tariromukute/trf-gen-cn5g:latest
-        entrypoint: /bin/bash -c \
-              "ip link set dev eth0 xdpgeneric obj /tmp/nsh-decap.bpf.o sec xdp_nsh_decap;"\
-              "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\
-              "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity"
-        command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"]
-        healthcheck:
-            test: /bin/bash -c "iptables -L -t nat | grep MASQUERADE"
-            interval: 10s
-            timeout: 5s
-            retries: 5
+        container_name: "n6-lan"
+        image: tariromukute/n6-lan-simple:latest
+        command: 
+        - /bin/bash
+        - -c
+        - |
+            sh testovs.sh
+            ovs-vsctl add-port brovs1 eth1
+            ip addr flush dev eth1 && ip addr add 192.168.72.138/26 dev brovs1 && ip link set brovs1 up
+            iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+            ip route add 12.1.1.0/24 via 192.168.72.134 dev brovs1
+            tail -f /dev/null
+        # command:  tail -f /dev/null
+        # command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"]
+        # apt-get install kmod
+        # network_mode: host
+        # pid: 'host'
+        # ovs-vsctl add-br brovs1
+        # ip route add default via 192.168.72.129 dev brovs1
+        devices:
+            - /dev/net/tun:/dev/net/tun # https://docs.openvswitch.org/en/stable/intro/install/userspace/#building-and-installing
+        volumes:
+            - /lib/modules:/lib/modules
         networks:
-            public_net:
-                ipv4_address: 192.168.70.135
+            # public_net:
+            #     ipv4_address: 192.168.70.138
             n6_net:
-                ipv4_address: 192.168.72.135
-                mac_address: 02:42:ac:11:65:44
-            # data_net:
-            #     ipv4_address: 192.168.73.135
-            #     mac_address: 02:42:ac:11:65:45
+                ipv4_address: 192.168.72.138
+            data_net:
+                ipv4_address: 192.168.73.138
+    # oai-ext-dn:
+    #     privileged: true
+    #     init: true
+    #     container_name: "oai-ext-dn"
+    #     image: tariromukute/trf-gen-cn5g:latest
+    #     entrypoint: /bin/bash -c \
+    #           "ip link set dev eth0 xdpgeneric obj /tmp/nsh-decap.bpf.o sec xdp_nsh_decap;"\
+    #           "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\
+    #           "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; ip route; sleep infinity"
+    #     command: ["/bin/bash", "-c", "trap : SIGTERM SIGINT; sleep infinity & wait"]
+    #     healthcheck:
+    #         test: /bin/bash -c "iptables -L -t nat | grep MASQUERADE"
+    #         interval: 10s
+    #         timeout: 5s
+    #         retries: 5
+    #     networks:
+    #         public_net:
+    #             ipv4_address: 192.168.70.135
+    #         n6_net:
+    #             ipv4_address: 192.168.72.135
+    #             mac_address: 02:42:ac:11:65:44
+    #         # data_net:
+    #         #     ipv4_address: 192.168.73.135
+    #         #     mac_address: 02:42:ac:11:65:45
 
     iperf3:
         privileged: true
         platform: linux/amd64
         container_name: "iperf3"
         image: ubuntu:jammy
-        command: tail -f /dev/null
+        # command: tail -f /dev/null
+        command: 
+        - /bin/bash
+        - -c
+        - |
+            apt update -y
+            apt install iperf3 -y
+            iperf3 -s
         cap_add:
             - NET_ADMIN
         networks:
-            n6_net:
-                ipv4_address: 192.168.72.137
+            data_net:
+                ipv4_address: 192.168.73.137
 
-    dropwatch:
-        privileged: true
-        platform: linux/amd64
-        container_name: "dropwatch"
-        image: ubuntu:jammy
-        command: tail -f /dev/null
-        volumes:
-        - /usr/src:/usr/src:ro 
-        - /lib/modules/:/lib/modules:ro 
-        - /sys/:/sys/:rw 
-        pid: 'host'
-        network_mode: host
+    # dropwatch:
+    #     privileged: true
+    #     platform: linux/amd64
+    #     container_name: "dropwatch"
+    #     image: ubuntu:jammy
+    #     command: tail -f /dev/null
+    #     volumes:
+    #     - /usr/src:/usr/src:ro 
+    #     - /lib/modules/:/lib/modules:ro 
+    #     - /sys/:/sys/:rw 
+    #     pid: 'host'
+    #     network_mode: host
 
 networks:
     public_net: