Merge pull request #110 from GaloisInc/97-ardupilot-build

Build ArduPilot in CI

This branch sets up CI to build the ArduPilot SITL binary for aarch64.  Specifically, this branch adds three things:

* Build scripts for setting up the build environment and compiling the `arduplane` binary
* An `ardupilot` section in `package.sh`
* A CI job to build and cache the `ardupilot` package.

.github/workflows/main.yml

@@ -410,3 +410,40 @@ jobs:
         rust-version: 1.74
     - name: Run MPS tests
       run: RUST_LOG=trace MPS_DEBUG=1 python3 src/vm_runner/tests/mps/run_tests.py
+
+  ardupilot:
+      runs-on: ubuntu-22.04
+      steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+        - name: Checkout submodules
+          run: |
+              git submodule update --init components/autopilot/ardupilot
+        - name: Hash inputs
+          id: hash
+          run: |
+              cache_key="$(bash src/pkvm_setup/package.sh cache_key ardupilot)"
+              echo "Cache key: $cache_key"
+              echo "CACHE_KEY=$cache_key" >>$GITHUB_OUTPUT
+              echo "CACHE_KEY=$cache_key" >>$GITHUB_ENV
+        - name: Cache results
+          id: cache
+          uses: actions/cache@v3
+          with:
+              key: ${{ env.CACHE_KEY }}
+              path: packages/${{ env.CACHE_KEY }}.tar.gz
+        - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+          name: Install dependencies
+          run: |
+            BUILD_ONLY=1 bash components/autopilot/ardupilot_install_deps.sh
+        - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+          name: Fetch additional submodules for build
+          run: |
+            bash components/autopilot/ardupilot_init_submodules.sh
+        - if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+          name: Build ArduPilot
+          run: |
+              bash src/pkvm_setup/package.sh full_build ardupilot
+      outputs:
+          CACHE_KEY: ${{ steps.hash.outputs.CACHE_KEY }}
+

.gitmodules

@@ -10,10 +10,10 @@
 	shallow = true
 [submodule "components/autopilot/ardupilot"]
 	path = components/autopilot/ardupilot
-	url = git@github.com:ArduPilot/ardupilot.git
+	url = https://github.com/ArduPilot/ardupilot.git
 [submodule "components/message_bus/czmq"]
 	path = components/message_bus/czmq
-	url = git@github.com:zeromq/czmq.git
+	url = https://github.com/zeromq/czmq.git
 [submodule "src/pkvm_setup/vhost-device"]
 	path = src/pkvm_setup/vhost-device
 	url = [email protected]:GaloisInc/verse-vhost-device.git

components/autopilot/ardupilot_build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build the ArduPilot SITL binary for aarch64.
+
+target=
+if [[ "$#" -ne 0 ]]; then
+    target="$1"
+fi
+
+build_dir=build
+if [[ -n "$target" ]]; then
+    build_dir="$build_dir.$target"
+fi
+
+cd "$(dirname "$0")/ardupilot"
+
+edo() {
+    echo " >> $*" 1>&2
+    "$@"
+}
+
+case "$target" in
+    aarch64)
+        export CC=aarch64-linux-gnu-gcc
+        export CXX=aarch64-linux-gnu-g++
+        export LD=aarch64-linux-gnu-g++
+        ;;
+esac
+
+. venv/bin/activate
+./waf -o "${build_dir}" configure --board sitl
+./waf -o "${build_dir}" build --target bin/arduplane

components/autopilot/ardupilot_init_submodules.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+
+# Initialize ArduPilot submodules that are needed for the SITL build.
+
+cd "$(dirname "$0")/ardupilot"
+
+edo() {
+    echo " >> $*" 1>&2
+    "$@"
+}
+
+modules=(
+    waf
+    DroneCAN/dronecan_dsdlc
+    DroneCAN/pydronecan
+    DroneCAN/DSDL
+    DroneCAN/libcanard
+)
+
+for x in "${modules[@]}"; do
+    edo git submodule update --init "modules/$x"
+done

components/autopilot/ardupilot_install_deps.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+set -euo pipefail
+
+# Script for installing ArduPilot build dependencies.  Run with `BUILD_ONLY=1`
+# to install only the dependencies required to build ArduPilot SITL binaries;
+# the default is to install these and also dependencies of the mavproxy ground
+# station software.
+
+cd "$(dirname "$0")/ardupilot"
+
+edo() {
+    echo " >> $*" 1>&2
+    "$@"
+}
+
+# Echo the first argument.  This is useful for expanding a (possible) glob
+# pattern to a single concrete filename.
+first() {
+    echo "$1"
+}
+
+# Install a package with `apt-get`, but only if a certain file is missing from
+# the system.  Running `apt-get install` on an already-installed package is a
+# no-op, but we'd like to avoid asking for `sudo` privileges unnecessarily.
+install_if_missing() {
+    local package="$1"
+    local file="$2"
+
+    # If `$2` is a glob pattern, expand it to the first matching file if one
+    # exists.
+    if [[ ! -f "$(IFS='' first $file)" ]]; then
+        echo "missing $file - need to install package $package" 1>&2
+        edo sudo apt-get install -y "$package"
+        if [[ ! -f "$(IFS='' first $file)" ]]; then
+            echo "actual contents of $package:" 1>&2
+            dpkg-query -L "$package" 1>&2
+            echo "error: expected package $package to provide $file, but it did not" 1>&2
+            return 1
+        fi
+    fi
+}
+
+# Create Python virtualenv if it doesn't yet exist.
+if [[ ! -d venv ]]; then
+    edo install_if_missing python3-virtualenv /usr/bin/virtualenv
+    virtualenv venv
+fi
+
+# Install Python dependencies into the virtualenv.  `pip install` is a no-op if
+# the package is already installed.
+(
+    . venv/bin/activate
+    pip3 install pexpect empy==3.3.4 future
+    if [[ -z "${BUILD_ONLY:+x}" ]]; then
+        pip3 install pymavlink MAVProxy opencv-python matplotlib
+
+        # Extra system package needed for building wxPython
+        edo install_if_missing libgtk-3-dev /usr/lib/*/pkgconfig/gtk+-3.0.pc
+        # Note: the wxPython install builds from source, which takes a while
+        pip3 install wxPython
+    fi
+)
+
+install_if_missing gcc-aarch64-linux-gnu /usr/bin/aarch64-linux-gnu-gcc
+install_if_missing g++-aarch64-linux-gnu /usr/bin/aarch64-linux-gnu-g++

src/pkvm_setup/package.sh

@@ -247,6 +247,28 @@ vm_images_list_outputs() {
 }
 
 
+# ardupilot
+
+ardupilot_get_input_hashes() {
+    (
+        cd components/autopilot
+        sha1sum ardupilot_build.sh
+        sha1sum ardupilot_init_submodules.sh
+        sha1sum ardupilot_install_deps.sh
+    )
+    ( cd components/autopilot/ardupilot && git rev-parse HEAD:./ )
+}
+
+ardupilot_build() {
+    bash components/autopilot/ardupilot_build.sh aarch64
+}
+
+ardupilot_list_outputs() {
+    echo components/autopilot/ardupilot/build.aarch64/sitl/bin/arduplane
+    echo components/autopilot/ardupilot/Tools/autotest/models/plane.parm
+}
+
+
 # Actions.  Each `do_foo` function can be called via `bash package.sh foo
 # package_name`.