From 11edc6390c1ab6aa58a44fcb79b392293ecab30b Mon Sep 17 00:00:00 2001 From: Julien Orain Date: Thu, 15 May 2025 15:14:05 +0000 Subject: [PATCH 1/6] azure: add Grok processor for AzureFirewallThreatIntelLog --- packages/azure/changelog.yml | 5 +++++ .../firewall_logs/elasticsearch/ingest_pipeline/default.yml | 6 ++++++ packages/azure/manifest.yml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 49bef61fbdb..6ad58da1eb1 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.24.0" + changes: + - description: Add Grok processor for `AzureFirewallThreatIntelLog` in `azure.firewall_logs`. + type: enhancement + link: https://github.com/elastic/integrations/pull/xx - version: "1.23.2" changes: - description: Fix Grok processor error in ingest pipeline for `AzureFirewallNetworkRuleLog` in `azure.firewall_logs`. diff --git a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml index c202a4d701e..8703bf7b9d1 100644 --- a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml @@ -194,6 +194,12 @@ processors: - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long}. Url: %{HOSTNAME:url.original}. Action: %{DATA:azure.firewall.action}. ThreatIntel: %{DATA:rule.name}$" if: ctx?.json?.operationName == 'AzureFirewallNetworkRuleLog' || ctx?.json?.operationName == 'AzureFirewallNatRuleLog' + - grok: + field: json.properties.msg + patterns: + - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long}. Action: %{DATA:azure.firewall.action}. ThreatIntel: %{DATA:rule.name}$" + if: ctx?.json?.operationName == 'AzureFirewallThreatIntelLog' + - grok: field: json.properties.msg patterns: diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 5e9bdeb4281..f8b7872c276 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.23.2" +version: "1.24.0" description: This Elastic integration collects logs from Azure type: integration icons: From 58d797573091791cf832ded5c727ac33121bdcc9 Mon Sep 17 00:00:00 2001 From: Julien Orain Date: Fri, 16 May 2025 09:29:08 +0200 Subject: [PATCH 2/6] Update packages/azure/changelog.yml Co-authored-by: Dan Kortschak --- packages/azure/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 6ad58da1eb1..f96222475d5 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -2,7 +2,7 @@ changes: - description: Add Grok processor for `AzureFirewallThreatIntelLog` in `azure.firewall_logs`. type: enhancement - link: https://github.com/elastic/integrations/pull/xx + link: https://github.com/elastic/integrations/pull/13921 - version: "1.23.2" changes: - description: Fix Grok processor error in ingest pipeline for `AzureFirewallNetworkRuleLog` in `azure.firewall_logs`. From 372544047c9416654dee77c08f2e050e805ab3d2 Mon Sep 17 00:00:00 2001 From: Julien Orain Date: Fri, 16 May 2025 08:16:16 +0000 Subject: [PATCH 3/6] add tests --- .../test/pipeline/test-threatintel-raw.log | 1 + .../test-threatintel-raw.log-expected.json | 75 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log create mode 100644 packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log new file mode 100644 index 00000000000..418bd3bd792 --- /dev/null +++ b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log @@ -0,0 +1 @@ +{"category":"AZFWThreatIntel","properties":{"Action":"Deny","DestinationIp":"175.16.199.1","DestinationPort":443,"Fqdn":"","IsTlsInspected":false,"Protocol":"TCP","SourceIp":"192.168.0.2","SourcePort":51890,"ThreatDescription":"Destination reported by Threat Intelligence","Url":""},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2025-05-16T07:34:42.525499+00:00"} diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json new file mode 100644 index 00000000000..1bba112e4f3 --- /dev/null +++ b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json @@ -0,0 +1,75 @@ +{ + "expected": [ + { + "@timestamp": "2025-05-16T07:34:42.525Z", + "azure": { + "firewall": { + "category": "AZFWThreatIntel", + "is_tls_inspected": false + }, + "resource": { + "group": "TEST-FW-RG", + "id": "/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01", + "name": "TEST-FW01", + "provider": "MICROSOFT.NETWORK/AZUREFIREWALLS" + }, + "subscription_id": "23103928-B2CF-472A-8CDB-0146E2849129" + }, + "cloud": { + "account": { + "id": "23103928-B2CF-472A-8CDB-0146E2849129" + }, + "provider": "azure" + }, + "destination": { + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.1", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Deny", + "category": [ + "network" + ], + "kind": "event", + "original": "{\"category\":\"AZFWThreatIntel\",\"properties\":{\"Action\":\"Deny\",\"DestinationIp\":\"175.16.199.1\",\"DestinationPort\":443,\"Fqdn\":\"\",\"IsTlsInspected\":false,\"Protocol\":\"TCP\",\"SourceIp\":\"192.168.0.2\",\"SourcePort\":51890,\"ThreatDescription\":\"Destination reported by Threat Intelligence\",\"Url\":\"\"},\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01\",\"time\":\"2025-05-16T07:34:42.525499+00:00\"}", + "type": [ + "connection" + ] + }, + "observer": { + "name": "TEST-FW01", + "product": "Network Firewall", + "type": "firewall", + "vendor": "Azure" + }, + "related": { + "ip": [ + "192.168.0.2", + "175.16.199.1" + ] + }, + "source": { + "ip": "192.168.0.2", + "port": 51890 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} From d6d440d29885799929cfe2a98f744b2db799fe49 Mon Sep 17 00:00:00 2001 From: Julien Orain Date: Mon, 2 Jun 2025 15:58:00 +0200 Subject: [PATCH 4/6] Update packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml Co-authored-by: Dan Kortschak --- .../firewall_logs/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml index 8703bf7b9d1..65ccaf012a9 100644 --- a/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml @@ -198,7 +198,7 @@ processors: field: json.properties.msg patterns: - "^%{DATA:azure.firewall.proto} request from %{IPORHOST:source.address}:%{NUMBER:source.port:long} to %{IPORHOST:destination.address}:%{NUMBER:destination.port:long}. Action: %{DATA:azure.firewall.action}. ThreatIntel: %{DATA:rule.name}$" - if: ctx?.json?.operationName == 'AzureFirewallThreatIntelLog' + if: ctx.json?.operationName == 'AzureFirewallThreatIntelLog' - grok: field: json.properties.msg From c834ce7f1a5c7cc2a889e834badab450319cca87 Mon Sep 17 00:00:00 2001 From: Julien Orain Date: Mon, 2 Jun 2025 14:18:49 +0000 Subject: [PATCH 5/6] fix tests --- .../test/pipeline/test-threatintel-raw.log | 2 +- .../test-threatintel-raw.log-expected.json | 50 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log index 418bd3bd792..2fac91ddc31 100644 --- a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log +++ b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log @@ -1 +1 @@ -{"category":"AZFWThreatIntel","properties":{"Action":"Deny","DestinationIp":"175.16.199.1","DestinationPort":443,"Fqdn":"","IsTlsInspected":false,"Protocol":"TCP","SourceIp":"192.168.0.2","SourcePort":51890,"ThreatDescription":"Destination reported by Threat Intelligence","Url":""},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2025-05-16T07:34:42.525499+00:00"} +{"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallThreatIntelLog","properties":{"msg":"TCP request from 192.168.0.2:49680 to 89.160.20.156:1688. Action: Deny. ThreatIntel: Destination reported by Threat Intelligence"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2025-05-16T07:34:42.525499+00:00"} \ No newline at end of file diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json index 1bba112e4f3..71cb809b120 100644 --- a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json +++ b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log-expected.json @@ -4,8 +4,9 @@ "@timestamp": "2025-05-16T07:34:42.525Z", "azure": { "firewall": { - "category": "AZFWThreatIntel", - "is_tls_inspected": false + "action": "Deny", + "category": "AzureFirewallNetworkRule", + "operation_name": "AzureFirewallThreatIntelLog" }, "resource": { "group": "TEST-FW-RG", @@ -22,35 +23,46 @@ "provider": "azure" }, "destination": { + "address": "89.160.20.156", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, "geo": { - "city_name": "Changchun", - "continent_name": "Asia", - "country_iso_code": "CN", - "country_name": "China", + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", "location": { - "lat": 43.88, - "lon": 125.3228 + "lat": 58.4167, + "lon": 15.6167 }, - "region_iso_code": "CN-22", - "region_name": "Jilin Sheng" + "region_iso_code": "SE-E", + "region_name": "Östergötland County" }, - "ip": "175.16.199.1", - "port": 443 + "ip": "89.160.20.156", + "port": 1688 }, "ecs": { "version": "8.11.0" }, "event": { - "action": "Deny", "category": [ "network" ], "kind": "event", - "original": "{\"category\":\"AZFWThreatIntel\",\"properties\":{\"Action\":\"Deny\",\"DestinationIp\":\"175.16.199.1\",\"DestinationPort\":443,\"Fqdn\":\"\",\"IsTlsInspected\":false,\"Protocol\":\"TCP\",\"SourceIp\":\"192.168.0.2\",\"SourcePort\":51890,\"ThreatDescription\":\"Destination reported by Threat Intelligence\",\"Url\":\"\"},\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01\",\"time\":\"2025-05-16T07:34:42.525499+00:00\"}", + "original": "{\"category\":\"AzureFirewallNetworkRule\",\"operationName\":\"AzureFirewallThreatIntelLog\",\"properties\":{\"msg\":\"TCP request from 192.168.0.2:49680 to 89.160.20.156:1688. Action: Deny. ThreatIntel: Destination reported by Threat Intelligence\"},\"resourceId\":\"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01\",\"time\":\"2025-05-16T07:34:42.525499+00:00\"}", "type": [ - "connection" + "connection", + "denied" ] }, + "network": { + "iana_number": "6", + "transport": "tcp" + }, "observer": { "name": "TEST-FW01", "product": "Network Firewall", @@ -60,12 +72,16 @@ "related": { "ip": [ "192.168.0.2", - "175.16.199.1" + "89.160.20.156" ] }, + "rule": { + "name": "Destination reported by Threat Intelligence" + }, "source": { + "address": "192.168.0.2", "ip": "192.168.0.2", - "port": 51890 + "port": 49680 }, "tags": [ "preserve_original_event" From 801882b46f1566726d9abc277cff69e188ca601f Mon Sep 17 00:00:00 2001 From: Julien Orain Date: Mon, 2 Jun 2025 22:12:44 +0000 Subject: [PATCH 6/6] fix review --- .../firewall_logs/_dev/test/pipeline/test-threatintel-raw.log | 2 +- packages/azure/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log index 2fac91ddc31..b45aa4cf62c 100644 --- a/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log +++ b/packages/azure/data_stream/firewall_logs/_dev/test/pipeline/test-threatintel-raw.log @@ -1 +1 @@ -{"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallThreatIntelLog","properties":{"msg":"TCP request from 192.168.0.2:49680 to 89.160.20.156:1688. Action: Deny. ThreatIntel: Destination reported by Threat Intelligence"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2025-05-16T07:34:42.525499+00:00"} \ No newline at end of file +{"category":"AzureFirewallNetworkRule","operationName":"AzureFirewallThreatIntelLog","properties":{"msg":"TCP request from 192.168.0.2:49680 to 89.160.20.156:1688. Action: Deny. ThreatIntel: Destination reported by Threat Intelligence"},"resourceId":"/SUBSCRIPTIONS/23103928-B2CF-472A-8CDB-0146E2849129/RESOURCEGROUPS/TEST-FW-RG/PROVIDERS/MICROSOFT.NETWORK/AZUREFIREWALLS/TEST-FW01","time":"2025-05-16T07:34:42.525499+00:00"} diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index f8b7872c276..293485bd15e 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: "1.24.0" +version: "1.25.0" description: This Elastic integration collects logs from Azure type: integration icons: