Migrate to using libbpf and BPF skeleton

.vscode/settings.json

@@ -1,6 +1,12 @@
 {
     "files.associations": {
         "socket.h": "c",
-        "parsing_helpers.h": "c"
+        "parsing_helpers.h": "c",
+        "tc-gtpu.h": "c",
+        "tcgtpu.h": "c",
+        "unistd.h": "c",
+        "signal.h": "c",
+        "gtpu.h": "c",
+        "tc-gtpu.skel.h": "c"
     }
 }

Dockerfile

@@ -1,16 +1,15 @@
 FROM ubuntu:latest
 
 RUN apt-get update && \
-    apt-get install -y clang llvm libbpf-dev iproute2 iputils-ping tcpdump && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get install -y clang llvm iproute2 iputils-ping tcpdump make git \
+    libelf1 libelf-dev zlib1g-dev gcc pkg-config 
+    # && \
+    # rm -rf /var/lib/apt/lists/*
 
 WORKDIR /home
 
-COPY gtpu.bpf.c ./
+COPY . ./
 
-RUN clang -O2 -emit-llvm -c gtpu.bpf.c -o - | llc -march=bpf -mcpu=probe -filetype=obj -o gtpu.bpf.o
+RUN make build
 
-# RUN tc qdisc add dev eth0 clsact
-# RUN tc filter add dev eth0 ingress bpf direct-action obj gtpu.bpf.o sec .text
-# RUN tc filter show dev eth0
-# RUN tc filter show dev eth0 ingress
+# ENTRYPOINT ./entrypoint.sh

Makefile

@@ -0,0 +1,5 @@
+build:
+	make -C src
+
+clean:
+	make -C src clean

NOTES.md

@@ -32,16 +32,13 @@ Either:
 1. create container and the mount the debugfs inside the container.
 
 ```bash
+# For dev: -v `pwd`/:/home \
 docker run \
     -it \
-    # --privileged \
     --cap-add=NET_ADMIN \
     --cap-add=SYS_ADMIN \
-    --device /dev/net/tun \
     -v /sys/:/sys/ \
-    # -v /lib/modules/:/lib/modules/:ro \
-    # -v /usr/src:/usr/src:ro \
-    -v `pwd`/:/home \
+    --device /dev/net/tun \
     tariromukute/tc-gtpu:latest
 
 mount -t debugfs debugfs /sys/kernel/debug
@@ -102,7 +99,7 @@ tcpdump -i eth0 -w tmp.pcap
 Or
 
 ```bash
-./gtpu_loader -g eth0 -i uegtp -s 172.0.0.1 -d 172.0.0.2 -u 12.1.1.2 -t 1234 -q 9 -n 2
+./tc-gtpu -g eth0 -i uegtp -s 172.0.0.1 -d 172.0.0.2 -u 12.1.1.2 -t 1234 -q 9 -n 2
 ```
 
 ```bash
@@ -114,4 +111,5 @@ docker run -it --rm --privileged --pid=host ubuntu:latest nsenter -t 1 -m -u -n
 - [Run eBPF Programs in Docker using docker-bpf](https://hemslo.io/run-ebpf-programs-in-docker-using-docker-bpf/)
 - https://github.com/edgecomllc/eupf/issues/509
 - http://arthurchiao.art/blog/differentiate-bpf-redirects/
+- https://patchwork.kernel.org/project/netdevbpf/patch/[email protected]/
 

entrypoint.sh

@@ -1,4 +1,4 @@
-#!/bin/env sh
+#!/bin/bash
 
 # Mount bpffs and debugfs if not present already
 if [[ $(/bin/mount | /bin/grep /sys/fs/bpf -c) -eq 0 ]]; then

src/.gitignore

@@ -0,0 +1,3 @@
+/.output
+/bootstrap
+/tc-gtpu

src/Makefile

@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+OUTPUT := .output
+CLANG ?= clang
+LLVM_STRIP ?= llvm-strip
+LIBBPF_SRC := $(abspath ../libbpf/src)
+BPFTOOL_SRC := $(abspath ../bpftool/src)
+LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
+BPFTOOL_OUTPUT ?= $(abspath $(OUTPUT)/bpftool)
+BPFTOOL ?= $(BPFTOOL_OUTPUT)/bootstrap/bpftool
+ARCH ?= $(shell uname -m | sed 's/x86_64/x86/' \
+			 | sed 's/aarch64/arm64/' \
+			 | sed 's/ppc64le/powerpc/' \
+			 | sed 's/mips.*/mips/' \
+			 | sed 's/arm.*/arm/' \
+			 | sed 's/riscv64/riscv/')
+VMLINUX := ../vmlinux/$(ARCH)/vmlinux.h
+# Use our own libbpf API headers and Linux UAPI headers distributed with
+# libbpf to avoid dependency on system-wide headers, which could be missing or
+# outdated
+INCLUDES := -I$(OUTPUT) -I../libbpf/include/uapi -I$(dir $(VMLINUX))
+CFLAGS := -g -Wall
+ALL_LDFLAGS := $(LDFLAGS) $(EXTRA_LDFLAGS)
+
+APPS = tc-gtpu
+
+# Get Clang's default includes on this system. We'll explicitly add these dirs
+# to the includes list when compiling with `-target bpf` because otherwise some
+# architecture-specific dirs will be "missing" on some architectures/distros -
+# headers such as asm/types.h, asm/byteorder.h, asm/socket.h, asm/sockios.h,
+# sys/cdefs.h etc. might be missing.
+#
+# Use '-idirafter': Don't interfere with include mechanics except where the
+# build would have failed anyways.
+CLANG_BPF_SYS_INCLUDES = $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+ifeq ($(V),1)
+	Q =
+	msg =
+else
+	Q = @
+	msg = @printf '  %-8s %s%s\n'					\
+		      "$(1)"						\
+		      "$(patsubst $(abspath $(OUTPUT))/%,%,$(2))"	\
+		      "$(if $(3), $(3))";
+	MAKEFLAGS += --no-print-directory
+endif
+
+define allow-override
+  $(if $(or $(findstring environment,$(origin $(1))),\
+            $(findstring command line,$(origin $(1)))),,\
+    $(eval $(1) = $(2)))
+endef
+
+$(call allow-override,CC,$(CROSS_COMPILE)cc)
+$(call allow-override,LD,$(CROSS_COMPILE)ld)
+
+.PHONY: all
+all: $(APPS)
+
+.PHONY: clean
+clean:
+	$(call msg,CLEAN)
+	$(Q)rm -rf $(OUTPUT) $(APPS)
+
+$(OUTPUT) $(OUTPUT)/libbpf $(BPFTOOL_OUTPUT):
+	$(call msg,MKDIR,$@)
+	$(Q)mkdir -p $@
+
+# Build libbpf
+$(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
+	$(call msg,LIB,$@)
+	$(Q)$(MAKE) -C $(LIBBPF_SRC) BUILD_STATIC_ONLY=1		      \
+		    OBJDIR=$(dir $@)/libbpf DESTDIR=$(dir $@)		      \
+		    INCLUDEDIR= LIBDIR= UAPIDIR=			      \
+		    install
+
+# Build bpftool
+$(BPFTOOL): | $(BPFTOOL_OUTPUT)
+	$(call msg,BPFTOOL,$@)
+	$(Q)$(MAKE) ARCH= CROSS_COMPILE= OUTPUT=$(BPFTOOL_OUTPUT)/ -C $(BPFTOOL_SRC) bootstrap
+
+# Build BPF code
+$(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(VMLINUX) | $(OUTPUT)
+	$(call msg,BPF,$@)
+	$(Q)$(CLANG) -g -fstack-protector -O2 -target bpf -D__TARGET_ARCH_$(ARCH) $(INCLUDES) $(CLANG_BPF_SYS_INCLUDES) -c $(filter %.c,$^) -o $@
+	$(Q)$(LLVM_STRIP) -g $@ # strip useless DWARF info
+
+# Generate BPF skeletons
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) $(BPFTOOL)
+	$(call msg,GEN-SKEL,$@)
+	$(Q)$(BPFTOOL) gen skeleton $< > $@
+
+# Build user-space code
+$(patsubst %,$(OUTPUT)/%.o,$(APPS)): %.o: %.skel.h
+
+$(OUTPUT)/%.o: %.c $(wildcard %.h) | $(OUTPUT)
+	$(call msg,CC,$@)
+	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
+
+# Build application binary
+$(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) | $(OUTPUT)
+	$(call msg,BINARY,$@)
+	$(Q)$(CC) $(CFLAGS) $^ $(ALL_LDFLAGS) -lelf -lz -o $@
+
+# delete failed targets
+.DELETE_ON_ERROR:
+
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:

gtpu.bpf.c → src/tc-gtpu.bpf.c

@@ -10,8 +10,7 @@
 #include <linux/udp.h>
 #include <linux/tcp.h>
 #include <linux/in.h>
-#include "gtpu.h"
-#include "config.h"
+#include "tc-gtpu.h"
 #include "parsing_helpers.h"
 
 #define TC_ACT_UNSPEC         (-1)
@@ -24,30 +23,24 @@
 #define __section(x) __attribute__((section(x), used))
 
 #define DEFAULT_QFI 9
-#define UDP_CSUM_OFF offsetof(struct udphdr, check)
-#define IP_CSUM_OFF (ETH_HLEN + offsetof(struct iphdr, check))
-#define IP_SRC_OFF (ETH_HLEN + offsetof(struct iphdr, saddr))
-#define IP_DST_OFF (ETH_HLEN + offsetof(struct iphdr, daddr))
-
-const int gtpu_interface  = 22;
-const __be32 gtpu_dest_ip = bpf_htonl(0xac110003); // 172.17.0.3
-const __be32 gtpu_src_ip = bpf_htonl(0xac110002); // 172.17.0.2
-
-struct ipv4_gtpu_encap {
-    struct iphdr ipv4h;
-    struct udphdr udp;
-    struct gtpuhdr gtpu;
-    struct gtpu_hdr_ext gtpu_hdr_ext;
-    struct gtp_pdu_session_container pdu;
-} __attribute__((__packed__));
-
-struct ipv6_gtpu_encap {
-    struct ipv6hdr ipv6h;
-    struct udphdr udp;
-    struct gtpuhdr gtpu;
-    struct gtpu_hdr_ext gtpu_hdr_ext;
-    struct gtp_pdu_session_container pdu;
-} __attribute__((__packed__));
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__type(key, __u32); // teid
+	__type(value, struct ingress_state);
+	__uint(max_entries, 32);
+    // __uint(pinning, LIBBPF_PIN_BY_NAME);
+} ingress_map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__type(key, __u32); // ifindex
+	__type(value, struct egress_state);
+	__uint(max_entries, 32);
+    // __uint(pinning, LIBBPF_PIN_BY_NAME);
+} egress_map SEC(".maps");
+
+const volatile struct gtpu_config config;
 
 static struct ipv4_gtpu_encap ipv4_gtpu_encap = {
 	.ipv4h.version = 4,
@@ -90,22 +83,6 @@ static struct ipv6_gtpu_encap ipv6_gtpu_encap = {
 	.pdu.next_ext = 0,
 };
 
-struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__type(key, __u32); // teid
-	__type(value, struct ingress_state);
-	__uint(max_entries, 32);
-    __uint(pinning, LIBBPF_PIN_BY_NAME);
-} ingress_map SEC(".maps");
-
-struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__type(key, __u32); // ifindex
-	__type(value, struct egress_state);
-	__uint(max_entries, 32);
-    __uint(pinning, LIBBPF_PIN_BY_NAME);
-} egress_map SEC(".maps");
-
 /* Logic for checksum, thanks to https://github.com/facebookincubator/katran/blob/main/katran/lib/bpf/csum_helpers.h */
 __attribute__((__always_inline__))
 static inline __u16 csum_fold_helper(__u64 csum) {
@@ -137,7 +114,7 @@ static inline void ipv4_csum_inline(
 }
 
 
-SEC("tnl_if_ingress")
+SEC("tc/ingress")
 int tnl_if_ingress_fn(struct __sk_buff *skb)
 {
     /**
@@ -162,7 +139,7 @@ int tnl_if_ingress_fn(struct __sk_buff *skb)
     }
 };
 
-SEC("tnl_if_egress")
+SEC("tc/egress")
 int tnl_if_egress_fn(struct __sk_buff *skb)
 {
     /**
@@ -214,8 +191,8 @@ int tnl_if_egress_fn(struct __sk_buff *skb)
         data = (void *)(unsigned long long)skb->data;
         eth = data;
 
-        ipv4_gtpu_encap.ipv4h.daddr = gtpu_dest_ip;
-        ipv4_gtpu_encap.ipv4h.saddr = gtpu_src_ip;
+        ipv4_gtpu_encap.ipv4h.daddr = config.daddr.addr.addr4.s_addr;
+        ipv4_gtpu_encap.ipv4h.saddr = config.saddr.addr.addr4.s_addr;
         ipv4_gtpu_encap.ipv4h.tot_len = bpf_htons(sizeof(struct ipv4_gtpu_encap) + payload_len);
 
         ipv4_gtpu_encap.udp.len = bpf_htons(sizeof(struct ipv4_gtpu_encap) + payload_len - sizeof(struct iphdr));
@@ -231,7 +208,7 @@ int tnl_if_egress_fn(struct __sk_buff *skb)
         }
 
         bpf_printk("Redirecting to gtpu interface\n");
-        return bpf_redirect_neigh(gtpu_interface, NULL, 0, 0);
+        return bpf_redirect_neigh(config.gtpu_ifindex, NULL, 0, 0);
 
     } else {
         bpf_printk("error: protocol not ETH_P_IP, it is: %d\n", eth->h_proto);
@@ -240,7 +217,7 @@ int tnl_if_egress_fn(struct __sk_buff *skb)
 }
 
 
-SEC("gtpu_ingress")
+SEC("tc/ingress")
 int gtpu_ingress_fn(struct __sk_buff *skb)
 {
     /**
@@ -300,7 +277,7 @@ int gtpu_ingress_fn(struct __sk_buff *skb)
     return TC_ACT_OK;
 };
 
-SEC("gtpu_egress")
+SEC("tc/egress")
 int gtpu_egress_fn(struct __sk_buff *skb)
 {
     /**