Merge remote-tracking branch 'origin/2.4/dev' into 2.4/navigator

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -22,6 +22,7 @@ body:
         - 2.4.90
         - 2.4.100
         - 2.4.110
+        - 2.4.111
         - 2.4.120
         - Other (please provide detail below)
     validations:

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.110-20241010 ISO image released on 2024/10/10
+### 2.4.111-20241217 ISO image released on 2024/12/18
 
 
 ### Download and Verify
 
-2.4.110-20241010 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.110-20241010.iso
+2.4.111-20241217 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.111-20241217.iso
 
-MD5: A8003DEBC4510D538F06238D9DBB86C0  
-SHA1: 441DE90A192C8FE8BEBAB9ACE1A3CC18F71A2B1F  
-SHA256: B087A0D12FC2CA3CCD02BD52E52421F4F60DC09BF826337A057E05A04D114CCE  
+MD5: 767823D75EB76A6DC6132F799FD0E720  
+SHA1: 0A7B6918FE5D4BC89EE3F2E03B4F8F4D6255141D  
+SHA256: 394BFCED9B5EAA0788E2D04806231B3A170839394AAF8DD23B4CE0EB9D6EF727  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.110-20241010.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.111-20241217.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.110-20241010.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.111-20241217.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.110-20241010.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.111-20241217.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.110-20241010.iso.sig securityonion-2.4.110-20241010.iso
+gpg --verify securityonion-2.4.111-20241217.iso.sig securityonion-2.4.111-20241217.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 10 Oct 2024 07:05:30 AM EDT using RSA key ID FE507013
+gpg: Signature made Tue 17 Dec 2024 04:33:10 PM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <[email protected]>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.4.120
+2.4.120

salt/docker/defaults.yaml

@@ -82,6 +82,7 @@ docker:
         - 443:443
         - 8443:8443
         - 7788:7788
+        - 7789:7789
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []

salt/elasticfleet/defaults.yaml

@@ -108,6 +108,7 @@ elasticfleet:
     - ti_anomali
     - ti_cybersixgill
     - ti_misp
+    - ti_opencti
     - ti_otx
     - ti_rapid7_threat_command
     - ti_recordedfuture

salt/elasticsearch/defaults.yaml

@@ -10353,6 +10353,52 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-ti_opencti_x_indicator:
+      index_sorting: False
+      index_template:
+        composed_of:
+          - "logs-ti_opencti.indicator@package"
+          - "logs-ti_opencti.indicator@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+        ignore_missing_component_templates:
+          - "logs-ti_opencti.indicator@custom"
+        index_patterns:
+          - "logs-ti_opencti.indicator-*"
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-ti_opencti.indicator-logs
+              number_of_replicas: 0
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 60d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-ti_otx_x_pulses_subscribed:
       index_sorting: false
       index_template:

salt/elasticsearch/files/ingest/zeek.quic

@@ -0,0 +1,18 @@
+{
+  "description" : "zeek.quic",
+  "processors" : [
+    { "set":      { "field": "event.dataset",                           "value": "quic" } },
+    { "set":      { "field": "network.transport",                       "value": "udp"  } },
+    { "json":     { "field": "message",                                 "target_field": "message2",                       "ignore_failure": true    } },
+    { "rename": 	{ "field": "message2.version",	                    "target_field": "quic.version",		              "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_initial_dcid",	        "target_field": "quic.client_initial_dcid",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_scid",	                "target_field": "quic.client_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_scid",	                "target_field": "quic.server_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_name",	                "target_field": "quic.server_name",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_protocol",	            "target_field": "quic.client_protocol",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.history",	                    "target_field": "quic.history",                   "ignore_missing": true 	} },
+    { "remove":		{ "field": "message2.tags",		                                                                      "ignore_failure": true    } },
+    { "remove":   { "field": ["host"],                                                                                    "ignore_failure": true    } },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -491,6 +491,7 @@ elasticsearch:
     so-logs-ti_cybersixgill_x_threat: *indexSettings
     so-logs-ti_misp_x_threat: *indexSettings
     so-logs-ti_misp_x_threat_attributes: *indexSettings
+    so-logs-ti_opencti_x_indicator: *indexSettings
     so-logs-ti_otx_x_pulses_subscribed: *indexSettings  
     so-logs-ti_otx_x_threat: *indexSettings
     so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings

salt/elasticsearch/templates/component/elastic-agent/logs-ti_opencti.indicator@custom.json

@@ -0,0 +1,36 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+	  "properties":{
+            "ip": {
+              "type": "ip"
+            }
+	  }
+	},
+	"related": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"destination": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"source": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/manager/tools/sbin/soup

@@ -404,7 +404,8 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
     [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100
     [[ "$INSTALLEDVERSION" == 2.4.100 ]] && up_to_2.4.110
-    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.120
+    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111
+    [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120
     true
 }
 
@@ -519,6 +520,11 @@ post_to_2.4.110() {
   POSTVERSION=2.4.110
 }
 
+post_to_2.4.111() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.111
+}
+
 post_to_2.4.120() {
   update_elasticsearch_index_settings
   POSTVERSION=2.4.120
@@ -714,6 +720,12 @@ up_to_2.4.110() {
   INSTALLEDVERSION=2.4.110
 }
 
+up_to_2.4.111() {
+  echo "Nothing to do for 2.4.111"
+
+  INSTALLEDVERSION=2.4.111
+}
+
 up_to_2.4.120() {
   add_hydra_pillars
 
@@ -944,7 +956,7 @@ update_airgap_rules() {
   rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
   rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
   # Copy the securityonion-resorces repo over for SOC Detection Summaries and checkout the published summaries branch
-  rsync -av --chown=socore:socore $UPDATE_DIR/agrules/securityonion-resources /opt/so/conf/soc/ai_summary_repos
+  rsync -av --delete --chown=socore:socore $UPDATE_DIR/agrules/securityonion-resources /opt/so/conf/soc/ai_summary_repos
   git config --global --add safe.directory /opt/so/conf/soc/ai_summary_repos/securityonion-resources
   git -C /opt/so/conf/soc/ai_summary_repos/securityonion-resources checkout generated-summaries-published
   # Copy the securityonion-resorces repo over to nsm

salt/soc/defaults.yaml

@@ -339,6 +339,16 @@ soc:
         - file.os
         - file.subsystem
         - log.id.fuid
+      '::quic':
+        - soc_timestamp
+        - event.dataset
+        - source.ip
+        - source.port
+        - destination.ip
+        - destination.port
+        - quic.server_name
+        - log.id.uid
+        - network.community_id
       '::radius':
         - soc_timestamp
         - event.dataset
@@ -1732,6 +1742,10 @@ soc:
               description: PE files list
               query: 'tags:pe | groupby file.machine file.os file.subsystem'
               showSubtitle: true
+            - name: QUIC
+              description: QUIC connections
+              query: 'tags:quic | groupby quic.server_name | groupby source.ip quic.server_name destination.ip'
+              showSubtitle: true
             - name: RADIUS
               description: RADIUS grouped by username
               query: 'tags:radius | groupby user.name'
@@ -1950,6 +1964,9 @@ soc:
             - name: PE
               description: PE (Portable Executable) files transferred via network traffic
               query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit'
+            - name: QUIC
+              description: QUIC network metadata
+              query: 'tags:quic | groupby quic.server_name | groupby -sankey quic.server_name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name | groupby quic.server_scid | groupby quic.version | groupby quic.client_protocol'
             - name: RADIUS
               description: RADIUS (Remote Authentication Dial-In User Service) network metadata
               query: 'tags:radius | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -45,46 +45,47 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
-    # Drops the Hashes field which is specific to Sysmon logs
-    # Ingested sysmon logs will have the Hashes field mapped to ECS specific fields
-    - id: hashes_drop_sysmon-specific-field
-      type: drop_detection_item
+    # Transforms the `Hashes` field to ECS fields
+    # ECS fields are used by the hash fields emitted by Elastic Defend
+    # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
+    - id: hashes_break_out_field
+      type: hashes_fields
+      valid_hash_algos: ["MD5", "SHA1", "SHA256", "SHA512", "IMPHASH"]
+      field_prefix: "file"
+      drop_algo_prefix: False
       field_name_conditions:
         - type: include_fields
           fields:
-          - winlog.event_data.Hashes
-      rule_conditions:
-      - type: logsource
-        product: windows
+            - winlog.event_data.Hashes
     - id: hashes_process-creation
       type: field_name_mapping
       mapping:
-        winlog.event_data.sha256: process.hash.sha256
-        winlog.event_data.sha1: process.hash.sha1
-        winlog.event_data.md5: process.hash.md5
-        winlog.event_data.Imphash: process.pe.imphash
+        fileSHA256: process.hash.sha256
+        fileSHA1: process.hash.sha1
+        fileMD5: process.hash.md5
+        fileIMPHASH: process.pe.imphash
       rule_conditions:
       - type: logsource
         product: windows
         category: process_creation
     - id: hashes_image-load
       type: field_name_mapping
       mapping:
-        winlog.event_data.sha256: dll.hash.sha256
-        winlog.event_data.sha1: dll.hash.sha1
-        winlog.event_data.md5: dll.hash.md5
-        winlog.event_data.Imphash: dll.pe.imphash
+        fileSHA256: dll.hash.sha256
+        fileSHA1: dll.hash.sha1
+        fileMD5: dll.hash.md5
+        fileIMPHASH: dll.pe.imphash
       rule_conditions:
       - type: logsource
         product: windows
         category: image_load
     - id: hashes_driver-load
       type: field_name_mapping
       mapping:
-        winlog.event_data.sha256: dll.hash.sha256
-        winlog.event_data.sha1: dll.hash.sha1
-        winlog.event_data.md5: dll.hash.md5
-        winlog.event_data.Imphash: dll.pe.imphash
+        fileSHA256: dll.hash.sha256
+        fileSHA1: dll.hash.sha1
+        fileMD5: dll.hash.md5
+        fileIMPHASH: dll.pe.imphash
       rule_conditions:
       - type: logsource
         product: windows

setup/so-functions

@@ -962,7 +962,12 @@ docker_seed_update() {
 docker_seed_registry() {
 	local VERSION="$SOVERSION"
 
-	if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then
+	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
+		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
+		logCmd "rm /nsm/docker-registry/docker/registry.tar"
+	elif [ -d /nsm/docker-registry/docker/registry ] && [ -f /etc/SOCLOUD ]; then
+		echo "Using existing docker registry content for cloud install"
+	else
 		if [ "$install_type" == 'IMPORT' ]; then
 			container_list 'so-import'	
 		else
@@ -972,9 +977,6 @@ docker_seed_registry() {
 		docker_seed_update_percent=25
 
 		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
-	else
-		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
-		logCmd "rm /nsm/docker-registry/docker/registry.tar"
 	fi
 }