Update to 5.1.3

Author: dkowalcz-sec

.gitignore

@@ -21,6 +21,7 @@ index.html
 .coverage
 
 images/iso/*.iso
+images/iso/local_files/
 packaging/output/
 packaging/root/opt/obsrvbl-ona/netflow/
 packaging/root/opt/obsrvbl-ona/ipfix/

Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ARCH ?= amd64
-VERSION := 5.1.2
+VERSION := 5.1.3
 
 SCRIPTS_DIR := src/scripts
 uPNA_DIR := src/uPNA
@@ -71,13 +71,13 @@ ona-service_RaspbianJessie_%.deb:
 	mkdir -p $(dir $@)
 	python package_builder.py $(notdir $*) ${VERSION} RaspbianJessie
 
-ona-service_UbuntuXenial_%.deb:
+ona-service_UbuntuNoble_%.deb:
 	mkdir -p $(dir $@)
-	python package_builder.py $(notdir $*) ${VERSION} UbuntuXenial
+	python package_builder.py $(notdir $*) ${VERSION} UbuntuNoble
 
-ona-service_UbuntuXenialContainer_%.deb:
+ona-service_UbuntuNobleContainer_%.deb:
 	mkdir -p $(dir $@)
-	python package_builder.py $(notdir $*) ${VERSION} UbuntuXenialContainer
+	python package_builder.py $(notdir $*) ${VERSION} UbuntuNobleContainer
 
 .PHONY: clean
 clean:

README.md

@@ -2,24 +2,28 @@
 
 This repository is where the development of the Observable Networks Appliance (ONA) takes place. The ONA software is used to collect input data for Observable Networks' network security service. It can run on a variety of platforms, including embedded computers, physical servers, virtual machines, cloud servers, and Docker containers.
 
-## Supported platforms
+## Download
 
-The following platforms are officially supported:
+### ISO (fully supported and recommended):
 
-* [Ubuntu 18.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb)
-* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_7_x86_64.rpm)
-* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_8_x86_64.rpm)
-* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_armhf.deb)
+* [Ubuntu 24.04](https://assets-production.obsrvbl.com/ona-packages/iso/ona-24.04.1-v5.1.3/ona-24.04.1-server-amd64.iso)
+
+### Package files for manual installation:
+
+* [Ubuntu 24.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb)
+* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_7_x86_64.rpm)
+* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_8_x86_64.rpm)
+* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_armhf.deb)
   ([installation guide](raspberry_pi_guide.md))
-* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_aarch64.deb)
+* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_aarch64.deb)
   ([installation guide](raspberry_pi_guide.md))
 * [Docker](https://github.com/obsrvbl/ona/blob/master/images/docker/Dockerfile)
 
-To install the latest version on 20.04 (recommended for physical and virtual machine installations):
+To install the latest version on Ubuntu:
 
 ```
-$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb
-$ sudo apt install ./ona-service_UbuntuXenial_amd64.deb
+$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb
+$ sudo apt install ./ona-service_UbuntuNoble_amd64.deb
 ```
 
 To monitor NetFlow traffic, you'll also need to install tools from the [CERT NetSA Security Suite](https://tools.netsa.cert.org/):

images/docker/Dockerfile

@@ -48,15 +48,15 @@ RUN curl -L -O https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/
     && rm -rf netsa-pkg.deb
 
 # Use local copy of ONA service package if needed
-# COPY ona-service_UbuntuXenialContainer_amd64.deb ./
+# COPY ona-service_UbuntuNobleContainer_amd64.deb ./
 
 # Install ONA service
-RUN if [ ! -f ./ona-service_UbuntuXenialContainer_amd64.deb ] ;\
-    then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenialContainer_amd64.deb ;\
+RUN if [ ! -f ./ona-service_UbuntuNobleContainer_amd64.deb ] ;\
+    then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNobleContainer_amd64.deb ;\
     else echo "Use cached package" ;fi \
-    && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuXenialContainer_amd64.deb \
+    && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuNobleContainer_amd64.deb \
     && rm -rf /var/lib/apt/lists/* \
-    && rm -rf ona-service_UbuntuXenialContainer_amd64.deb
+    && rm -rf ona-service_UbuntuNobleContainer_amd64.deb
 
 # Switch to the unprivileged user, set some local configuration, and start.
 COPY run.sh /opt/obsrvbl-ona/run.sh

images/iso/build_iso.sh

@@ -21,9 +21,9 @@
 #    wrong.
 #
 
-RELEASE="${RELEASE:-20.04.1}"
+RELEASE="${RELEASE:-24.04.1}"
 ARCH="${ARCH:-amd64}"
-VARIANT="${VARIANT:-legacy}"
+VARIANT="${VARIANT:-subiquity}"
 
 
 DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
@@ -45,47 +45,138 @@ while getopts "f:a:r:" opt ; do
            ;;
     esac
 done
+# Newly added
+ ubuntu_name="ubuntu-${RELEASE}-live-server-${ARCH}.iso"
+ ona_name="ona-${RELEASE}-server-${ARCH}.iso"
+ ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}"
+
+# ubuntu_name="ubuntu-24.04.1-live-server-amd64.iso"
+# ona_name="ona-${RELEASE}-server-${ARCH}.iso"
+ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/"
+if [ -n "$PUBLIC_ONA" ]; then
+  ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/"
+fi
+# netsa_pkg_name="netsa-pkg.deb"
+ona_pkg_name="ona-service_UbuntuNoble_amd64.deb"
 
-ubuntu_name="ubuntu-${RELEASE}-server-${ARCH}.iso"
-ona_name="ona-${RELEASE}-server-${ARCH}.iso"
-ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}"
 test -n "$ubuntu_url" || fatal "failed getting Ubuntu ISO download URL"
-ona_service_url="https://s3.amazonaws.com/onstatic/ona-service/master/ona-service_UbuntuXenial_amd64.deb"
-netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb"
+
+ ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/"
+ if [ -n "$PUBLIC_ONA" ]; then
+   ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/"
+
+ fi
+
+ #ona_service_url="${ONA_URL}ona-service_UbuntuNoble_amd64.deb"
+ ona_service_url="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb"
+ netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb"
+
+
 
 shift $(($OPTIND-1))
 
 test $EUID -ne 0 && sudo="sudo"
-which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
-which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
 
 [[ -d "$DIR" ]] || fatal  # invalid directory
-[[ -d "$DIR"/working && $(ls -A "$DIR"/working) ]] && fatal  # working directory exists and is not empty
 [[ -d "$DIR"/working ]] || mkdir "$DIR"/working # working directory does not exist, so create it
+
+major_version=$(echo "$RELEASE" | cut -d '.' -f 1)
+
+# Check if the major version number is greater than 20
+if [ "$major_version" -gt 20 ]; then
+  which xorriso 1> /dev/null || fatal "missing xorriso: $sudo apt-get install xorriso -y"
+  NEW_FORMAT=true
+  BOOT_CAT="/boot.catalog"
+  EFI='/boot/grub/i386-pc/eltorito.img'
+  ELTORITO='/boot/grub/i386-pc/eltorito.img'
+else
+  which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
+  which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
+  BOOT_CAT="isolinux/boot.cat"
+  EFI="isolinux/isolinux.bin"
+  ELTORITO="boot/grub/efi.img"
+fi
+
 (
   set -e
+  if [ ! -e "/root/$ubuntu_name" ]; then
+    curl -L -o /root/${ubuntu_name} "${ubuntu_url}"
+  fi
+
   cd "$DIR"/working
-  curl -L -o ${ubuntu_name} "${ubuntu_url}"
+  #[[ -d "$DIR/local_files/" ]] && cp "$DIR"/local_files/* .
   curl -L -o netsa-pkg.deb "${netsa_pkg_url}"
-  curl -L -o ona-service.deb "${ona_service_url}"
+  #curl -L -o "${ona_pkg_name}" "${ona_service_url}"
+  $sudo cp /obsrvbl/images/iso/ona-service_UbuntuNoble_amd64.deb /obsrvbl/images/iso/working/
+
+
+
+
+
+$sudo apt-get -y update
+# you can install packages here if you want
+
+PACKAGES="apt-transport-https iptables-persistent ipset libjansson4 libltdl7 liblzo2-2 libnet1 libyaml-0-2 nano ntp ntpdate snmp tcpdump net-tools libsnappy1v5 python3-dateutil"
+$sudo apt-get -yyqq install --download-only ${PACKAGES}
+
+
+
+
+  # local is root dir in ISO
   mkdir cdrom local
-  $sudo mount -o loop --read-only "${ubuntu_name}" cdrom
+  pwd
+
+  $sudo mount -o loop --read-only "/root/${ubuntu_name}" cdrom
   rsync -av --quiet cdrom/ local
-  $sudo cp ../preseed/* local/preseed/
+
+  $sudo cp -r /var/cache/apt local
   $sudo cp -r ../ona local
   $sudo cp netsa-pkg.deb local/ona/netsa-pkg.deb
-  $sudo cp ona-service.deb local/ona/ona-service.deb
-  $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg
-  $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
-  $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \
-          -cache-inodes \
-          -J -l -b isolinux/isolinux.bin \
-          -c isolinux/boot.cat -no-emul-boot \
-          -boot-load-size 4 -boot-info-table \
-          -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
-          -o "../${ona_name}" local
+  $sudo cp ${ona_pkg_name} local/ona/${ona_pkg_name}
+
+  echo "New format: $NEW_FORMAT "
+  if [ -n "$NEW_FORMAT" ]; then
+    # copy autoinstall folders for grub
+    $sudo cp -r ../autoinstall/nocloud-dhcp  local/
+    $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
+  else
+    $sudo cp ../preseed/* local/preseed/
+    $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg
+    $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
+  fi
+
+  if [ -n "$NEW_FORMAT" ]; then
+    xorriso -as mkisofs -r  -V 'SWC Sensor Install CD' \
+      -o "../${ona_name}"\
+      --grub2-mbr --interval:local_fs:0s-15s:zero_mbrpt,zero_gpt:"/root/${ubuntu_name}" \
+      -partition_offset 16 \
+      --mbr-force-bootable \
+      -append_partition 2 0xef \
+      --interval:local_fs:4099440d-4109507d::"/root/${ubuntu_name}" \
+      -appended_part_as_gpt \
+      -c "${BOOT_CAT}" \
+      -b "${ELTORITO}" \
+      -no-emul-boot -boot-load-size 4 -boot-info-table \
+      --grub2-boot-info \
+      -eltorito-alt-boot \
+      -e '--interval:appended_partition_2:::' \
+      -no-emul-boot \
+      local
+  else
+    $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \
+      -cache-inodes \
+      -J -l -b "${BOOT_CAT}" \
+      -c "${EFI}" -no-emul-boot \
+      -joliet-long \
+      -boot-load-size 4 -boot-info-table \
+      -eltorito-alt-boot -e "${ELTORITO}" -no-emul-boot \
+      -o "../${ona_name}" local
+
+    isohybrid "../${ona_name}"
+  fi
+
   $sudo umount cdrom
   $sudo chown $USER:$USER "../${ona_name}"
-  isohybrid "../${ona_name}"
+  $sudo rm -rf "$DIR"/working
 )
-$sudo rm -rf "$DIR"/working
+

images/iso/build_iso_helper

@@ -17,6 +17,12 @@ $ ./build_iso_helper --doctests
 
 Real results as of February 2022:
 
+>>> get_iso_url('24.04', subiquity=True)
+https://releases.ubuntu.com/noble/ubuntu-24.04.1-live-server-amd64.iso
+
+>>> get_iso_url('22.04', subiquity=True)
+'https://releases.ubuntu.com/jammy/ubuntu-22.04.4-live-server-amd64.iso'
+
 >>> get_iso_url('20.04.3', subiquity=True)
 'https://releases.ubuntu.com/20.04/ubuntu-20.04.3-live-server-amd64.iso'
 
@@ -53,9 +59,10 @@ ValueError: no release found
 Traceback (most recent call last):
 ValueError: no Subiquity release for this version
 """
+
 from argparse import ArgumentParser, Action, SUPPRESS
 from doctest import DocTestSuite
-from unittest import TestCase, TextTestRunner, makeSuite
+from unittest import TestCase, TextTestRunner, TestLoader
 from unittest.mock import patch
 from urllib.request import Request, build_opener, HTTPRedirectHandler
 from urllib.error import URLError, HTTPError
@@ -120,11 +127,14 @@ def get_iso_url(version, subiquity: bool, arch='amd64'):
             urlpath = f'https://releases.ubuntu.com/{v.short}/'
             isofile = f'ubuntu-{v.long_if_patch}-live-server-{arch}.iso'
         else:
-            urlpath = (
-                'https://cdimage.ubuntu.com/ubuntu-legacy-server/releases'
-                f'/{v.short}/release/'
-            )
-            isofile = f'ubuntu-{v.long_if_patch}-legacy-server-{arch}.iso'
+            if v.major >= 22:
+                raise ValueError(f'No legacy ISO for version {v.short}')
+            else:
+              urlpath = (
+                  'https://cdimage.ubuntu.com/ubuntu-legacy-server/releases'
+                  f'/{v.short}/release/'
+              )
+              isofile = f'ubuntu-{v.long_if_patch}-legacy-server-{arch}.iso'
     elif v.major >= 18:
         if subiquity:
             urlpath = f'https://releases.ubuntu.com/{v.short}/'
@@ -181,7 +191,7 @@ class Tests(TestCase):
 
         def fake_head_status_code(url):
             if url in (
-                f'{cd_legacy}/20.04.1/release/ubuntu-20.04.1-legacy-server-amd64.iso',
+                f'{cd_legacy}/20.04/release/ubuntu-20.04.1-legacy-server-amd64.iso',
                 f'{releases}/20.04/ubuntu-20.04.1-live-server-amd64.iso',
                 f'{old}/20.04.0/ubuntu-20.04-live-server-amd64.iso',
                 'https://www.google.com',
@@ -195,9 +205,10 @@ class Tests(TestCase):
                 get_iso_url('20.04.1', True),
                 f'{releases}/20.04/ubuntu-20.04.1-live-server-amd64.iso',
             )
+
             self.assertEqual(
                 get_iso_url('20.04.1', False),
-                f'{cd_legacy}/20.04.1/release/ubuntu-20.04.1-legacy-server-amd64.iso',
+                f'{cd_legacy}/20.04/release/ubuntu-20.04.1-legacy-server-amd64.iso',
             )
             self.assertEqual(
                 get_iso_url('20.04.0', True),
@@ -210,7 +221,7 @@ class Tests(TestCase):
 class _TestAction(Action):
     def __init__(self, option_strings, dest, default=SUPPRESS, help=None):
         if dest == 'unittests':
-            self._suite = makeSuite(Tests)
+            self._suite = TestLoader().loadTestsFromTestCase(Tests)
         elif dest == 'doctests':
             self._suite = DocTestSuite()
         else:

images/iso/isolinux/grub.cfg

@@ -1,23 +1,36 @@
-
 if loadfont /boot/grub/font.pf2 ; then
-	set gfxmode=auto
-	insmod efi_gop
-	insmod efi_uga
-	insmod gfxterm
-	terminal_output gfxterm
+    set gfxmode=auto
+    insmod efi_gop
+    insmod efi_uga
+    insmod gfxterm
+    terminal_output gfxterm
 fi
 
 set menu_color_normal=white/black
 set menu_color_highlight=black/light-gray
 
 set timeout=30
+
+loadfont unicode
+
 menuentry "Install ONA (Static IP)" {
-	set gfxpayload=keep
-	linux	/install/vmlinuz  file=/cdrom/preseed/nodhcp.seed quiet ---
-	initrd	/install/initrd.gz
-}
-menuentry "Install ONA (DHCP)" {
-	set gfxpayload=keep
-	linux	/install/vmlinuz  file=/cdrom/preseed/dhcp.seed quiet ---
-	initrd	/install/initrd.gz
+    set gfxpayload=keep
+    linux   /casper/vmlinuz debug autoinstall ds=nocloud\;s=/cdrom/nocloud-dhcp/ ---
+    initrd  /casper/initrd
 }
+
+
+
+grub_platform
+if [ "$grub_platform" = "efi" ]; then
+    menuentry 'Boot from next volume' {
+        exit 1
+    }
+    menuentry 'UEFI Firmware Settings' {
+    zo    fwsetup
+    }
+else
+    menuentry 'Test memory' {
+        linux16 /boot/memtest86+.bin
+    }
+fi