diff --git a/azure-iptables-monitor/README.md b/azure-iptables-monitor/README.md index 48aa3b060b..682d30c2b1 100644 --- a/azure-iptables-monitor/README.md +++ b/azure-iptables-monitor/README.md @@ -25,14 +25,20 @@ Follow the steps below to build and run the program: 4. Start the program with: ```bash - ./azure-iptables-monitor --input=/etc/config/ --interval=300 + ./azure-iptables-monitor -input=/etc/config/ -interval=300 ``` - - The `--input` flag specifies the directory containing allowed regex pattern files. Default: `/etc/config/` - - The `--interval` flag specifies how often to check iptables rules in seconds. Default: `300` - - The `--events` flag enables Kubernetes event creation for rule violations. Default: `false` + - The `-input` flag specifies the directory containing allowed regex pattern files. Default: `/etc/config/` + - The `-input6` flag specifies the directory containing allowed regex pattern files for IPv6 ip6tables. Default: `/etc/config6/` + - The `-interval` flag specifies how often to check iptables rules and the bpf map in seconds. Default: `300` + - The `-events` flag enables Kubernetes event creation for rule violations. Default: `false` + - The `-ipv6` flag enables IPv6 ip6tables monitoring using the IPv6 allowlists. Default: `false` + - The `-checkMap` flag enables checking the pinned bpf map specified in mapPath for increases. Default: `false` + - The `-mapPath` flag specifies the pinned bpf map path to check. Default: `/azure-block-iptables/iptables_block_event_counter` - The program must be in a k8s environment and `NODE_NAME` must be a set environment variable with the current node. -5. The program will set the `user-iptables-rules` label to `true` on the specified ciliumnode resource if unexpected rules are found, or `false` if all rules match expected patterns. Proper RBAC is required for patching (patch for ciliumnodes, create for events, get for nodes). +5. The program will set the `kubernetes.azure.com/user-iptables-rules` label to `true` on the specified ciliumnode resource if unexpected rules are found, or `false` if all rules match expected patterns. Proper RBAC is required for patching (patch for ciliumnodes, create for events, get for nodes). + +6. The program will also send out an event if the bpf map value specified increases between checks ## Pattern File Format @@ -48,6 +54,7 @@ Each pattern file should contain one regex pattern per line: - `nat`, `mangle`, `filter`, `raw`, `security`: Patterns specific to each iptables table - Empty lines are ignored - Each line should be a valid Go regex pattern +- The ipv6 config directory uses files with same names, but will match against ipv6 iptables rules ## Debugging diff --git a/azure-iptables-monitor/go.mod b/azure-iptables-monitor/go.mod index 04f8a416f0..f5cb17a4f5 100644 --- a/azure-iptables-monitor/go.mod +++ b/azure-iptables-monitor/go.mod @@ -16,6 +16,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cilium/ebpf v0.19.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect diff --git a/azure-iptables-monitor/go.sum b/azure-iptables-monitor/go.sum index 561bcf8e39..4fc10e9187 100644 --- a/azure-iptables-monitor/go.sum +++ b/azure-iptables-monitor/go.sum @@ -4,6 +4,8 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.19.0 h1:Ro/rE64RmFBeA9FGjcTc+KmCeY6jXmryu6FfnzPRIao= +github.com/cilium/ebpf v0.19.0/go.mod h1:fLCgMo3l8tZmAdM3B2XqdFzXBpwkcSTroaVqN08OWVY= github.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNsPRc= github.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/azure-iptables-monitor/iptables_monitor.go b/azure-iptables-monitor/iptables_monitor.go index 9ccdc07acf..2e3452f49a 100644 --- a/azure-iptables-monitor/iptables_monitor.go +++ b/azure-iptables-monitor/iptables_monitor.go @@ -10,6 +10,7 @@ import ( "regexp" "time" + "github.com/cilium/ebpf" goiptables "github.com/coreos/go-iptables/iptables" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,12 +28,16 @@ import ( var version string var ( - configPath = flag.String("input", "/etc/config/", "Name of the directory with the allowed regex files") - checkInterval = flag.Int("interval", 300, "How often to check iptables rules (in seconds)") + configPath4 = flag.String("input", "/etc/config/", "Name of the directory with the ipv4 allowed regex files") + configPath6 = flag.String("input6", "/etc/config6/", "Name of directory with the ipv6 allowed regex files") + checkInterval = flag.Int("interval", 300, "How often to check for user iptables rules and bpf map increases (in seconds)") sendEvents = flag.Bool("events", false, "Whether to send node events if unexpected iptables rules are detected") + ipv6Enabled = flag.Bool("ipv6", false, "Whether to check ip6tables using the ipv6 allowlists") + checkMap = flag.Bool("checkMap", false, "Whether to check the bpf map at mapPath for increases") + pinPath = flag.String("mapPath", "/azure-block-iptables/iptables_block_event_counter", "Path to pinned bpf map") ) -const label = "user-iptables-rules" +const label = "kubernetes.azure.com/user-iptables-rules" type FileLineReader interface { Read(filename string) ([]string, error) @@ -197,10 +202,10 @@ func hasUnexpectedRules(currentRules, allowedPatterns []string) bool { // nodeHasUserIPTablesRules returns true if the node has iptables rules that do not match the regex // specified in the rule's respective table: nat, mangle, filter, raw, or security // The global file's regexes can match to a rule in any table -func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTablesClient) bool { +func nodeHasUserIPTablesRules(fileReader FileLineReader, path string, iptablesClient IPTablesClient) bool { tables := []string{"nat", "mangle", "filter", "raw", "security"} - globalPatterns, err := fileReader.Read(filepath.Join(*configPath, "global")) + globalPatterns, err := fileReader.Read(filepath.Join(path, "global")) if err != nil { globalPatterns = []string{} klog.V(2).Infof("No global patterns file found, using empty patterns") @@ -208,6 +213,8 @@ func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTables userIPTablesRules := false + klog.V(2).Infof("Using reference patterns files in %s", path) + for _, table := range tables { rules, err := GetRules(iptablesClient, table) if err != nil { @@ -216,7 +223,7 @@ func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTables } var referencePatterns []string - referencePatterns, err = fileReader.Read(filepath.Join(*configPath, table)) + referencePatterns, err = fileReader.Read(filepath.Join(path, table)) if err != nil { referencePatterns = []string{} klog.V(2).Infof("No reference patterns file found for table %s", table) @@ -234,6 +241,23 @@ func nodeHasUserIPTablesRules(fileReader FileLineReader, iptablesClient IPTables return userIPTablesRules } +func getBPFMapValue() (uint64, error) { + m, err := ebpf.LoadPinnedMap(*pinPath, nil) + if err != nil { + return 0, fmt.Errorf("failed to load pinned map %s: %w", *pinPath, err) + } + defer m.Close() + // 0 is the key for # of blocks + key := uint32(0) + value := uint64(0) + + if err := m.Lookup(&key, &value); err != nil { + return 0, fmt.Errorf("failed to lookup key %d in bpf map: %w", key, err) + } + + return value, nil +} + func main() { klog.InitFlags(nil) flag.Parse() @@ -263,6 +287,15 @@ func main() { klog.Fatalf("failed to create iptables client: %v", err) } + var ip6tablesClient IPTablesClient + if *ipv6Enabled { + ip6tablesClient, err = goiptables.New(goiptables.IPFamily(goiptables.ProtocolIPv6)) + if err != nil { + klog.Fatalf("failed to create ip6tables client: %v", err) + } + } + klog.Infof("IPv6: %v", *ipv6Enabled) + // get current node name from environment variable currentNodeName := os.Getenv("NODE_NAME") if currentNodeName == "" { @@ -273,8 +306,22 @@ func main() { var fileReader FileLineReader = OSFileLineReader{} + previousBlocks := uint64(0) + for { - userIPTablesRulesFound := nodeHasUserIPTablesRules(fileReader, iptablesClient) + userIPTablesRulesFound := nodeHasUserIPTablesRules(fileReader, *configPath4, iptablesClient) + if userIPTablesRulesFound { + klog.Info("Above user iptables rules detected in IPv4 iptables") + } + + // check ip6tables rules if enabled + if *ipv6Enabled { + userIP6TablesRulesFound := nodeHasUserIPTablesRules(fileReader, *configPath6, ip6tablesClient) + if userIP6TablesRulesFound { + klog.Info("Above user iptables rules detected in IPv6 iptables") + } + userIPTablesRulesFound = userIPTablesRulesFound || userIP6TablesRulesFound + } // update label based on whether user iptables rules were found err = patchLabel(dynamicClient, userIPTablesRulesFound, currentNodeName) @@ -291,6 +338,28 @@ func main() { } } + // if disabled the number of blocks never increases from zero + currentBlocks := uint64(0) + if *checkMap { + // read bpf map to check for number of blocked iptables rules + currentBlocks, err = getBPFMapValue() + if err != nil { + klog.Errorf("failed to get bpf map value: %v", err) + } + klog.V(2).Infof("IPTables rules blocks: Previous: %d Current: %d", previousBlocks, currentBlocks) + } + // if number of blocked rules increased since last time + blockedRulesIncreased := currentBlocks > previousBlocks + if *sendEvents && blockedRulesIncreased { + msg := "A process attempted to add iptables rules to the node but was blocked since last check. " + + "iptables rules blocked because EBPF Host Routing is enabled: aka.ms/acnsperformance" + err = createNodeEvent(clientset, currentNodeName, "BlockedIPTablesRule", msg, corev1.EventTypeWarning) + if err != nil { + klog.Errorf("failed to create iptables block event: %v", err) + } + } + previousBlocks = currentBlocks + time.Sleep(time.Duration(*checkInterval) * time.Second) } } diff --git a/azure-iptables-monitor/iptables_monitor_test.go b/azure-iptables-monitor/iptables_monitor_test.go index 2ebbfa27ab..60f025b7b6 100644 --- a/azure-iptables-monitor/iptables_monitor_test.go +++ b/azure-iptables-monitor/iptables_monitor_test.go @@ -217,7 +217,7 @@ func TestNodeHasUserIPTablesRules(t *testing.T) { fileReader.files = tc.files iptablesClient.rules = tc.rules - result := nodeHasUserIPTablesRules(fileReader, iptablesClient) + result := nodeHasUserIPTablesRules(fileReader, "/etc/config/", iptablesClient) require.Equal(t, tc.expected, result, tc.description) }) } diff --git a/bpf-prog/azure-block-iptables/pkg/bpfprogram/program.go b/bpf-prog/azure-block-iptables/pkg/bpfprogram/program.go index a69167b044..1033fac47d 100644 --- a/bpf-prog/azure-block-iptables/pkg/bpfprogram/program.go +++ b/bpf-prog/azure-block-iptables/pkg/bpfprogram/program.go @@ -18,7 +18,7 @@ import ( const ( // BPFMapPinPath is the directory where BPF maps are pinned - BPFMapPinPath = "/sys/fs/bpf/block-iptables" + BPFMapPinPath = "/sys/fs/bpf/azure-block-iptables" // EventCounterMapName is the name used for pinning the event counter map EventCounterMapName = "iptables_block_event_counter" // IptablesLegacyBlockProgramName is the name used for pinning the legacy iptables block program