Merge pull request #441 from Security-Onion-Solutions/2.4/dev

2.4.5

so-mysql/Dockerfile

@@ -12,7 +12,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
-FROM ghcr.io/security-onion-solutions/rockylinux:9
+FROM ghcr.io/security-onion-solutions/oraclelinux:9
 
 LABEL maintainer "Security Onion Solutions, LLC"
 LABEL description="MySQL Server running in Docker container for use with Security Onion"

so-pcaptools/Dockerfile

@@ -19,11 +19,14 @@ FROM ghcr.io/security-onion-solutions/python:3-slim
 ADD evtx_calc_timestamps.sh /evtx_calc_timestamps.sh
 RUN chmod +x /evtx_calc_timestamps.sh
 
+ADD timeshift.py /timeshift.py
+RUN chmod +x /timeshift.py
+
 # libwiretap11 is required for capinfo libaries
 # jq is required for evtx timestamp script
 RUN apt-get update && apt-get install -y --no-install-recommends --force-yes libwiretap11 git jq && rm -rf /var/lib/apt/lists/*
 RUN pip3 install evtx elasticsearch==7.17.1 tqdm orjson importlib_metadata
 RUN pip3 install evtx2es --no-dependencies
 
 COPY --from=builder /usr/bin/pcapfix /usr/bin/
-COPY --from=builder /usr/bin/capinfos /usr/bin/
+COPY --from=builder /usr/bin/capinfos /usr/bin/

so-pcaptools/evtx_calc_timestamps.sh

@@ -3,8 +3,15 @@
 # Convert EVTX to JSON
 evtx2json -q "/tmp/data.evtx" --output-file /tmp/evtx/import.json
 
-# Re-format JSON so that it is line-delimited
+# Check for timeshift
+if [[ -z "${SHIFTTS}" ]]; then
+# Ensure JSON is line-delimited
 cat /tmp/evtx/import.json | jq -c .[] > /tmp/evtx/data.json
+else
+# Shift timestamp
+python timeshift.py /tmp/evtx/import.json "${SHIFTTS}" event.created
+cat /tmp/evtx/import.json | jq -c .[] > /tmp/evtx/data.json
+fi
 
 # Remove older import file
 [ -f /tmp/evtx/import.json ] && rm -f /tmp/evtx/import.json

so-pcaptools/timeshift.py

@@ -0,0 +1,83 @@
+import json
+import argparse
+from datetime import datetime, timedelta
+
+def parse_custom_date(date_str):
+    # Add additional date formats as needed
+    custom_formats = ["%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S.%fZ"]
+    for fmt in custom_formats:
+        try:
+            return datetime.strptime(date_str, fmt)
+        except ValueError:
+            pass
+    # If none of the custom formats match, try parsing with ISO 8601 format
+    return datetime.fromisoformat(date_str.rstrip("Z"))
+
+def shift_timestamp(json_data, reference_date_str, nested_key):
+    # Parse the reference date string to a datetime object
+    reference_date = parse_custom_date(reference_date_str)
+
+    # Find the most recent date for the specified nested key and calculate the time shift
+    valid_dates = [parse_custom_date(find_nested_value(data, nested_key)) for data in json_data if find_nested_value(data, nested_key)]
+    most_recent_date = max(valid_dates) if valid_dates else reference_date
+    time_shift = reference_date - most_recent_date
+
+    # Shift the dates for the specified nested key based on the calculated time_shift
+    for data in json_data:
+        try:
+            date_str = find_nested_value(data, nested_key)
+            if date_str:
+                date_datetime = parse_custom_date(date_str)
+                shifted_datetime = date_datetime + time_shift
+
+                # Update the '@timestamp' and event.created fields with the shifted timestamp value
+                data['@timestamp'] = shifted_datetime.isoformat() + "Z"
+                data['timestamp'] = shifted_datetime.isoformat() + "Z"
+                data['event']['created'] = shifted_datetime.isoformat() + "Z"
+                data['winlog']['event_data']['UtcTime'] = shifted_datetime.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
+        except KeyError:
+            pass
+
+    return json_data
+
+def find_nested_value(data, nested_key):
+    keys = nested_key.split('.')
+    value = data
+    for key in keys:
+        if key in value:
+            value = value[key]
+        else:
+            raise KeyError("Key not found: {}".format(nested_key))
+    return value
+
+def read_json_data_from_file(file_path):
+    with open(file_path, "r") as file:
+        json_data = json.load(file)
+    return json_data
+
+def write_jsonl_data_to_file(file_path, json_data):
+    with open(file_path, "w") as file:
+        json.dump(json_data, file)
+
+def main():
+    # Parse command-line arguments
+    parser = argparse.ArgumentParser(description="Shift the @timestamp field for Windows Event Logs")
+    parser.add_argument("file_path", help="Path to the file containing JSON data (JSON array).")
+    parser.add_argument("reference_date", nargs="?", default=None, help="Reference date in custom format (e.g., '2023-08-01T16:00:00Z').")
+    parser.add_argument("nested_key", help="Nested key in the JSON objects containing the timestamp to be shifted (e.g., 'event.created').")
+    args = parser.parse_args()
+
+    # Read JSON data from the file
+    json_data = read_json_data_from_file(args.file_path)
+
+    # If the reference date is not provided as an argument, set it to the current time
+    reference_date_str = args.reference_date if args.reference_date else datetime.utcnow().isoformat() + "Z"
+
+    # Call the function to shift the '@timestamp' field relative to the reference date
+    shifted_json_data = shift_timestamp(json_data, reference_date_str, args.nested_key)
+
+    # Save the updated JSON data back to the file
+    write_jsonl_data_to_file("/tmp/evtx/import.json", shifted_json_data)
+
+if __name__ == "__main__":
+    main()

so-soctopus/so-soctopus/playbook/securityonion-baseline.yml

@@ -216,7 +216,7 @@ logsources:
 defaultindex: "*:so-*"
 fieldmappings:
     #START: SO Specific Mappings
-    DestinationIsIpv6: 
+    logtype: event.code
     EventID: event.code
     Channel: winlog.channel
     Protocol: network.transport
@@ -687,4 +687,4 @@ fieldmappings:
     ApplicationPath: winlog.event_data.ApplicationPath
     ModifyingApplication: winlog.event_data.ModifyingApplication
     Action: winlog.event_data.Action
-    #END: Default WLB/ECS Mappings
+    #END: Default WLB/ECS Mappings

so-strelka-backend/Dockerfile

@@ -19,7 +19,7 @@ ARG CAPA_VERSION=5.0.0
 ARG EXIFTOOL_VERSION=12.52
 
 # SO - Pin to release tag, download from GitHub, and prepare container dirs
-ARG STRELKA_RELEASE_VERSION=0.23.07.04
+ARG STRELKA_RELEASE_VERSION=0.23.07.10
 RUN mkdir /strelka && \
     mkdir /etc/strelka && \
     mkdir /tmp/strelka && \
@@ -183,7 +183,7 @@ USER $USERNAME
 # Run build checks
 RUN echo '[+] Run build checks' && \
     cd /strelka/strelka/ && \
-    python3 -m pytest -p no:cacheprovider -s tests/ --ignore-glob='*pcap*.py' && \
+    python3 -m pytest -p no:cacheprovider -s tests/ --ignore-glob='*pcap*.py' --ignore-glob='*test_scan_transcode.py' --ignore-glob='*test_scan_nf.py' && \
     if $CONFIG_TESTS; then python3 -m pytest -s tests_configuration/; fi && \
     echo '[+] Done'
 

so-strelka-filestream/Dockerfile

@@ -1,6 +1,6 @@
 FROM ghcr.io/security-onion-solutions/golang:alpine AS build
 LABEL maintainer "Security Onion Solutions, LLC"
-ARG STRELKA_RELEASE_VERSION=0.23.07.04
+ARG STRELKA_RELEASE_VERSION=0.23.07.10
 
 RUN CGO_ENABLED=0 go install github.com/target/strelka/src/go/cmd/strelka-filestream@$STRELKA_RELEASE_VERSION
 

so-strelka-frontend/Dockerfile

@@ -1,6 +1,6 @@
 FROM ghcr.io/security-onion-solutions/golang AS build
 LABEL maintainer "Security Onion Solutions, LLC"
-ARG STRELKA_RELEASE_VERSION=0.23.07.04
+ARG STRELKA_RELEASE_VERSION=0.23.07.10
 
 RUN CGO_ENABLED=0 go install github.com/target/strelka/src/go/cmd/strelka-frontend@$STRELKA_RELEASE_VERSION
 

so-strelka-manager/Dockerfile

@@ -1,6 +1,6 @@
 FROM ghcr.io/security-onion-solutions/golang AS build
 LABEL maintainer "Security Onion Solutions, LLC"
-ARG STRELKA_RELEASE_VERSION=0.23.07.04
+ARG STRELKA_RELEASE_VERSION=0.23.07.10
 
 RUN CGO_ENABLED=0 go install github.com/target/strelka/src/go/cmd/strelka-manager@$STRELKA_RELEASE_VERSION
 

so-suricata/Dockerfile

@@ -13,29 +13,38 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM ghcr.io/security-onion-solutions/centos:7 as builder
-
-RUN yum -y install epel-release
-RUN yum -y install jansson-devel libpcap-devel python3 libpcap-devel openssl-devel zlib-devel jemalloc-devel python3-devel kernel-devel kernel-headers libjansson libgeoip liblua5.1 curl wget make gcc pkg-config libhiredis libevent pcre-devel libpcre libmagic zlib libyaml rustc cargo libyaml-devel libcap-ng-devel file-devel nss-devel nspr-devel python3-yaml luajit-devel luajit
-
+FROM ghcr.io/security-onion-solutions/oraclelinux:9 as builder
+
+RUN yum -y install epel-release && \
+    dnf config-manager --enable ol9_codeready_builder
+
+RUN dnf -y install oraclelinux-developer-release-el9 
+RUN dnf repolist
+RUN dnf -y install autoconf automake diffutils file-devel gcc gcc-c++ git \
+               jansson-devel jq libcap-ng-devel libevent-devel \
+               libmaxminddb-devel libnet-devel libnetfilter_queue-devel \
+               libnfnetlink-devel libpcap-devel libtool libyaml-devel \
+               lua-devel lz4-devel make nss-devel pcre-devel pcre2-devel pkgconfig \
+               python3-devel python3-sphinx python3-yaml sudo which cargo \
+               zlib-devel luajit-devel cargo && cargo install --force cbindgen
 ENV SURIVERSION=6.0.13
 RUN mkdir /suricata
 
 WORKDIR /suricata
 
 RUN curl -vO https://www.openinfosecfoundation.org/download/suricata-$SURIVERSION.tar.gz && tar zxvf suricata-$SURIVERSION.tar.gz && \
-    cd suricata-$SURIVERSION && ./configure --enable-rust --enable-luajit --prefix=/opt/suricata --sysconfdir=/etc --disable-gccmarch-native --localstatedir=/var && make
+    cd suricata-$SURIVERSION && ./configure --enable-rust --enable-luajit --prefix=/opt/suricata --sysconfdir=/etc --disable-gccmarch-native --localstatedir=/var && make -j4
 RUN mkdir suriinstall && cd suricata-$SURIVERSION && make install DESTDIR=/suricata/suriinstall && make install-conf DESTDIR=/suricata/suriinstall && rm -rf /suricata/suriinstall/var/run
 
-FROM ghcr.io/security-onion-solutions/centos:7
+FROM ghcr.io/security-onion-solutions/oraclelinux:9
 
 LABEL maintainer "Security Onion Solutions, LLC"
 LABEL description="Suricata running in a docker with AF_Packet for use with Security Onion."
 
 COPY --from=builder /suricata/suriinstall/ /
 
 RUN yum -y install epel-release bash libpcap iproute && \
-    yum -y install GeoIP luajit libnet jansson libyaml cargo rustc && \
+    yum -y install luajit libnet jansson libyaml cargo rustc nss nss-devel && \
     yum -y erase epel-release && yum clean all && rm -rf /var/cache/yum && \
     groupadd --gid 940 suricata && \
     adduser --uid 940 --gid 940 --home-dir /etc/suricata --no-create-home suricata && \

so-tcpreplay/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright Security Onion Solutions, LLC
+# Copyright 2014-2023 Security Onion Solutions, LLC
 
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
@@ -40,4 +40,3 @@ rm *.deb && \
 rm *.rpm
 
 ENTRYPOINT ["/bin/bash"]
-

so-zeek/Dockerfile

@@ -13,10 +13,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-FROM ghcr.io/security-onion-solutions/rockylinux:9 as builder 
+FROM ghcr.io/security-onion-solutions/oraclelinux:9 as builder 
 
 RUN dnf -y install dnf-plugins-core && \
-    dnf config-manager --set-enabled crb && \
+    dnf config-manager --enable ol9_codeready_builder && \
     dnf update -y && \ 
     dnf -y install epel-release bash libpcap iproute wget cmake swig && \
     dnf -y install jemalloc numactl libnl3 libdnet gdb git && \
@@ -85,7 +85,7 @@ RUN groupadd --gid 937 zeek  && \
     chown -R 937:937 /nsm/zeek && \
     chown -R 937:937 /usr/local/ssl
 
-FROM ghcr.io/security-onion-solutions/rockylinux:9
+FROM ghcr.io/security-onion-solutions/oraclelinux:9
 
 LABEL maintainer "Security Onion Solutions, LLC"
 LABEL description="Zeek running in a docker with AF_Packet 3.2.0 for use with Security Onion"