diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 736375e1..f14af916 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -127,48 +127,12 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install SDK extensions - run: | - # XXX: slc-cli does not actually work when the extensions aren't in the SDK! - for sdk in /*_sdk_*; do - slc signature trust --sdk "$sdk" - - ln -s $PWD/gecko_sdk_extensions "$sdk"/extension - - for ext in "$sdk"/extension/*/; do - slc signature trust --sdk "$sdk" --extension-path "$ext" - done - done - - name: Build firmware id: build-firmware run: | - # Fix `fatal: detected dubious ownership in repository at` - git config --global --add safe.directory "$GITHUB_WORKSPACE" - - # Pass all SDKs as consecutive `--sdk ...` arguments - sdk_args="" - for sdk_dir in /*_sdk*; do - sdk_args="$sdk_args --sdk $sdk_dir" - done - - # Pass all toolchains as consecutive `--toolchain ...` arguments - toolchain_args="" - for toolchain_dir in /opt/*arm-none-eabi*; do - toolchain_args="$toolchain_args --toolchain $toolchain_dir" - done - - # Build it - /opt/venv/bin/python3 tools/build_project.py \ - $sdk_args \ - $toolchain_args \ - --manifest "${{ matrix.manifest }}" \ - --build-dir build \ - --build-system makefile \ - --output-dir outputs \ - --output gbl \ - --output hex \ - --output out + build_firmware.sh \ + --work-dir "$GITHUB_WORKSPACE" \ + --manifest "${{ matrix.manifest }}" # Get the basename of the GBL in `outputs` output_basename=$(basename -- $(basename -- $(ls -1 outputs/*.gbl | head -n 1)) .gbl) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml new file mode 100644 index 00000000..00a79987 --- /dev/null +++ b/.github/workflows/shellcheck.yml @@ -0,0 +1,34 @@ +name: reviewdog / shellscripts + +on: + push: + paths: + - tools/build_firmware.sh + + pull_request: + paths: + - tools/build_firmware.sh + +jobs: + + shellcheck: + name: Check shell scripts with shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: shellcheck + uses: reviewdog/action-shellcheck@v1 + with: + reporter: github-pr-review + path: tools + pattern: "*.sh" + + shfmt: + name: Check shell scripts with shfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: reviewdog/action-shfmt@v1 + with: + level: warning + shfmt_flags: '-i 0 ' \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7a24addf..f65b1652 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ autogen/ *.project.mak *.Makefile build/ +outputs/ +build_dir/ artifact/ artifacts/ trashed_modified_files/ diff --git a/Dockerfile b/Dockerfile index 68ecd16c..b41172f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ RUN \ xz-utils COPY requirements.txt /tmp/ +COPY tools/build_firmware.sh /usr/bin/build_firmware.sh RUN \ virtualenv /opt/venv \ @@ -30,8 +31,10 @@ RUN \ # Install Simplicity Commander (unfortunately no stable URL available, this # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz). +ARG SIMPLICITY_COMMANDER_SHA256SUM=ce7b9138c54f4fa0a24c48c8347e55e3e5f8b402d7f32615771bd0403c2d8962 RUN \ curl -O https://www.silabs.com/documents/login/software/SimplicityCommander-Linux.zip \ + && echo "${SIMPLICITY_COMMANDER_SHA256SUM} SimplicityCommander-Linux.zip" | sha256sum -c \ && unzip -q SimplicityCommander-Linux.zip \ && tar -C /opt -xjf SimplicityCommander-Linux/Commander_linux_x86_64_*.tar.bz \ && rm -r SimplicityCommander-Linux \ @@ -40,34 +43,44 @@ RUN \ ENV PATH="$PATH:/opt/commander" # Install Silicon Labs Configurator (slc) +ARG SLC_CLI_SHA256SUM=da4faa09ef4cbe385da71e5b95a4e444666cf4aaca6066b1095ca13bf5ebf233 RUN \ curl -O https://www.silabs.com/documents/login/software/slc_cli_linux.zip \ + && echo "${SLC_CLI_SHA256SUM} slc_cli_linux.zip" | sha256sum -c \ && unzip -q -d /opt slc_cli_linux.zip \ && rm slc_cli_linux.zip ENV PATH="$PATH:/opt/slc_cli" # GCC Embedded Toolchain 12.2.rel1 (for Gecko SDK 4.4.0+) +ARG GCC_TOOLCHAIN_SHA256SUM=84be93d0f9e96a15addd490b6e237f588c641c8afdf90e7610a628007fc96867 RUN \ curl -O https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu/12.2.rel1/binrel/arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz \ + && echo "${GCC_TOOLCHAIN_SHA256SUM} arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz" | sha256sum -c \ && tar -C /opt -xf arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz \ && rm arm-gnu-toolchain-12.2.rel1-x86_64-arm-none-eabi.tar.xz # Simplicity SDK 2024.6.2 +ARG SIMPLICITY_SDK_SHA256SUM=7e4337c7cc68262dd3a83c8528095774634a0478d40b1c1fd2b462e86236af8a RUN \ curl -o simplicity_sdk_2024.6.2.zip -L https://github.com/SiliconLabs/simplicity_sdk/releases/download/v2024.6.2/gecko-sdk.zip \ + && echo "${SIMPLICITY_SDK_SHA256SUM} simplicity_sdk_2024.6.2.zip" | sha256sum -c \ && unzip -q -d simplicity_sdk_2024.6.2 simplicity_sdk_2024.6.2.zip \ && rm simplicity_sdk_2024.6.2.zip # Gecko SDK 4.4.4 +ARG GECKO_SDK_SHA256SUM=831ec7c564df4392b18a8cc8ceb228c114dc3bec604be75807961a4289ee9b20 RUN \ curl -o gecko_sdk_4.4.4.zip -L https://github.com/SiliconLabs/gecko_sdk/releases/download/v4.4.4/gecko-sdk.zip \ + && echo "${GECKO_SDK_SHA256SUM} gecko_sdk_4.4.4.zip" | sha256sum -c \ && unzip -q -d gecko_sdk_4.4.4 gecko_sdk_4.4.4.zip \ && rm gecko_sdk_4.4.4.zip # ZCL Advanced Platform (ZAP) v2024.09.27 +ARG ZAP_SHA256SUM=22beeae3cf33b04792be379261d68695b5c96986d3b80700c22b1348f4c0421e RUN \ curl -o zap_2024.09.27.zip -L https://github.com/project-chip/zap/releases/download/v2024.09.27/zap-linux-x64.zip \ + && echo "${ZAP_SHA256SUM} zap_2024.09.27.zip" | sha256sum -c \ && unzip -q -d /opt/zap zap_2024.09.27.zip \ && rm zap_2024.09.27.zip @@ -81,5 +94,12 @@ ARG USER_GID=$USER_UID RUN groupadd --gid $USER_GID $USERNAME \ && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME +RUN mkdir -p /build_dir /outputs +RUN chown $USERNAME:$USERNAME \ + /gecko_sdk_* \ + /simplicity_sdk_* \ + /build_dir \ + /outputs + USER $USERNAME WORKDIR /build diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..1d13d44e --- /dev/null +++ b/Makefile @@ -0,0 +1,75 @@ +TOPDIR = $(shell pwd) +DOCKER_CHECK := $(shell command -v docker 2> /dev/null) +PODMAN_CHECK := $(shell command -v podman 2> /dev/null) + +ifdef PODMAN_CHECK + CONTAINER_ENGINE ?= podman +else ifdef DOCKER_CHECK + CONTAINER_ENGINE ?= docker +else + $(error "No container engine found, please install docker or podman") +endif + +CONTAINER_NAME ?= silabs-firmware-builder +CONTAINER_USER_GROUP ?= $(shell id -u):$(shell id -g) + +ifeq ($(CONTAINER_ENGINE),docker) + VOLUME_OPTS= +else + VOLUME_OPTS=:z +endif + +define run_in_container + $(CONTAINER_ENGINE) \ + run --rm \ + --user $(CONTAINER_USER_GROUP) \ + -v $(TOPDIR):/build$(VOLUME_OPTS) \ + -v $(TOPDIR)/outputs:/outputs$(VOLUME_OPTS) \ + -v $(TOPDIR)/build_dir:/build_dir$(VOLUME_OPTS) \ + $(CONTAINER_NAME) +endef + +MANIFESTS ?= $(shell find manifests -type f \( -name "*.yaml" -o -name "*.yml" \) -print) + +all: build_container build_firmware + +help: + @echo "Usage: make [all|build_container|build_firmware]" + @echo "" + @echo "Targets:" + @echo " all Build container and firmware" + @echo " build_container Build container" + @echo " build_firmware Build firmware" + @echo " help Show this help message" + @echo "" + @echo "Options:" + @echo " build_firmware MANIFESTS= Override default manifest files (default: all .yaml/.yml files in manifests/)" + @echo "" + @echo "Examples:" + @echo " # Build the container image" + @echo " make build_container" + @echo "" + @echo " # Build all firmware manifests" + @echo " make build_firmware" + @echo "" + @echo " # Build a specific firmware manifest" + @echo " make build_firmware MANIFESTS=manifests/nabucasa/yellow_bootloader.yaml" + @echo "" + +./outputs ./build_dir: + mkdir -p $@ +ifneq ($(CONTAINER_ENGINE),docker) + $(CONTAINER_ENGINE) unshare chown -R $(shell id -u):$(shell id -g) $@ +endif + +build_container: + $(CONTAINER_ENGINE) build -t $(CONTAINER_NAME) . + +build_firmware: ./outputs ./build_dir + $(run_in_container) \ + bash -c " \ + build_firmware.sh \ + --build-dir /build_dir \ + --output-dir /outputs \ + $(foreach manifest,$(MANIFESTS),--manifest $(manifest)) \ + " diff --git a/README.md b/README.md index dfc212c5..b7e5ccc3 100644 --- a/README.md +++ b/README.md @@ -90,3 +90,117 @@ python tools/build_project.py \ ``` Once the build is complete, the firmwares will be in the `output` directory. + +## Building with a container (for development) + +This is a convenience GNU Make based wrapper around the build process that is being used on the GitHub Actions CI pipeline, but can also be used for local development. + +### Prerequisites + +- [GNU Make](https://www.gnu.org/software/make/) +- [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman-desktop.io/) + +| Prerequisite | macOS | Windows | Debian/Ubuntu | Fedora | +|-------------|--------|---------|---------------|---------| +| GNU Make | `brew install make` | Via [Chocolatey](https://chocolatey.org/): `choco install make` | `sudo apt install make` | `sudo dnf install make` | +| Docker | Download [Docker Desktop](https://www.docker.com/products/docker-desktop/) | Download [Docker Desktop](https://www.docker.com/products/docker-desktop/) | `sudo apt install docker.io` | `sudo dnf install docker-ce docker-ce-cli containerd.io` | +| Podman | `brew install podman` | Download [Podman Desktop](https://podman-desktop.io/downloads) | `sudo apt install podman` | `sudo dnf install podman` | + +### Usage + +#### Help + +Provides a list of commands and options. + +```bash +make help + +Usage: make [all|build_container|build_firmware] + +Targets: + all Build container and firmware + build_container Build container + build_firmware Build firmware + help Show this help message + +Options: + build_firmware MANIFESTS= Override default manifest files (default: all .yaml/.yml files in manifests/) + +Examples: + # Build the container image + make build_container + + # Build all firmware manifests + make build_firmware + + # Build a specific firmware manifest + make build_firmware MANIFESTS=manifests/nabucasa/yellow_bootloader.yaml +``` + +#### Build everything + +Builds the container image and all available firmware manifests. + +```bash +make +``` + +Once this command completes, the firmwares will be in the `outputs` directory. + +```bash +ls -w 80 outputs | head -3 +skyconnect_bootloader_2.4.2.gbl +skyconnect_bootloader_2.4.2.hex +skyconnect_bootloader_2.4.2.out +``` + +#### Build the container + +Builds only the container image. + +```bash +make build_container +``` + +#### Build all available firmware manifests + +Builds all available firmware manifests in the `manifests` directory. + +```bash +make build_firmware +``` + +#### Build a specific firmware manifest + +Builds a specific firmware manifest by providing the path to the manifest file. + +```bash +make build_firmware MANIFESTS=manifests/nabucasa/yellow_openthread_ncp.yaml +``` + +Once this command completes, the firmwares will be in the `outputs` directory. + +```bash +ls -w 80 outputs +yellow_openthread_rcp_2.4.4.0_GitHub-7074a43e4_gsdk_4.4.4.gbl +yellow_openthread_rcp_2.4.4.0_GitHub-7074a43e4_gsdk_4.4.4.hex +yellow_openthread_rcp_2.4.4.0_GitHub-7074a43e4_gsdk_4.4.4.out +``` + +#### Build with a custom container image + +Builds the firmware with a custom container image by providing the container image name. + +```bash +make build_firmware CONTAINER_NAME=ghcr.io/nabucasa/silabs-firmware-builder +``` + +### Makefile variables + +The following variables can be customized when running make commands: + +| Variable | Default Value | Description | +|----------|---------------|-------------| +| CONTAINER_NAME | silabs-firmware-builder | Name of the container image to build/use | +| CONTAINER_ENGINE | docker | Container engine to use (docker or podman) | +| MANIFESTS | every file in `manifests` directory| Which firmware manifests to build | \ No newline at end of file diff --git a/tools/build_firmware.sh b/tools/build_firmware.sh new file mode 100755 index 00000000..36ee8ea5 --- /dev/null +++ b/tools/build_firmware.sh @@ -0,0 +1,140 @@ +#!/bin/bash +set -euo pipefail + +build_dir="build_dir" +output_dir="outputs" +work_dir="/build" +manifest_files=() +python_interpreter="/opt/venv/bin/python3" + +# Show help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo + echo "Build firmware images from manifest files" + echo + echo "Options:" + echo " -h, --help Show this help message" + echo " -m, --manifest FILE YAML manifest file describing the firmware to build" + echo " (can be specified multiple times)" + echo " -b, --build-dir DIR Directory for build files (default: ${build_dir})" + echo " -o, --output-dir DIR Directory for output files (default: ${output_dir})" + echo " -w, --work-dir DIR Working directory for build process (default: ${work_dir})" + echo " -p, --python PATH Python interpreter path (default: ${python_interpreter})" + exit 0 +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h | --help) + show_help + ;; + -m | --manifest) + if [ -z "${2:-}" ]; then + echo "Error: --manifest requires a file argument" + exit 1 + fi + manifest_files+=("$2") + shift + ;; + -b | --build-dir) + if [ -z "${2:-}" ]; then + echo "Error: --build-dir requires a directory argument" + exit 1 + fi + build_dir="$2" + shift + ;; + -o | --output-dir) + if [ -z "${2:-}" ]; then + echo "Error: --output-dir requires a directory argument" + exit 1 + fi + output_dir="$2" + shift + ;; + -w | --work-dir) + if [ -z "${2:-}" ]; then + echo "Error: --work-dir requires a directory argument" + exit 1 + fi + work_dir="$2" + shift + ;; + -p | --python) + if [ -z "${2:-}" ]; then + echo "Error: --python requires a path argument" + exit 1 + fi + python_interpreter="$2" + shift + ;; + *) + echo "Error: Unknown argument: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac + shift +done + +# Check that Python3 with ruamel.yaml is available +if ! "$python_interpreter" -c "import ruamel.yaml" &>/dev/null; then + echo "Error: Python3 with ruamel.yaml is not available at $python_interpreter" + echo "Install ruamel.yaml with 'pip install ruamel.yaml'" + exit 1 +fi + +# Check if any manifest files were provided +if [ ${#manifest_files[@]} -eq 0 ]; then + echo "Error: No manifest files provided" + echo "Use -h or --help for usage information" + exit 1 +fi + +git config --global --add safe.directory "$work_dir" + +# Install SDK extensions +for sdk in /*_sdk_*; do + slc signature trust --sdk "$sdk" + ln -s "$(pwd)"/gecko_sdk_extensions "$sdk"/extension + for ext in "$sdk"/extension/*/; do + slc signature trust --sdk "$sdk" --extension-path "$ext" + done +done + +# Build SDK arguments +sdk_args=() +for sdk_dir in /*_sdk*; do + sdk_args+=(--sdk "$sdk_dir") +done + +# Build toolchain arguments +toolchain_args=() +for toolchain_dir in /opt/*arm-none-eabi*; do + toolchain_args+=(--toolchain "$toolchain_dir") +done + +# Determine if we should keep the SLC daemon running +keep_daemon="" +if [ ${#manifest_files[@]} -gt 1 ]; then + keep_daemon="--keep-slc-daemon" +fi + +# Build firmware +for manifest_file in "${manifest_files[@]}"; do + echo "Building firmware manifest $manifest_file, using build directory $build_dir and output directory $output_dir" + + "$python_interpreter" tools/build_project.py \ + "${sdk_args[@]}" \ + "${toolchain_args[@]}" \ + $keep_daemon \ + --manifest "$manifest_file" \ + --build-dir "$build_dir" \ + --output-dir "$output_dir" \ + --build-system makefile \ + --output gbl \ + --output hex \ + --output out +done diff --git a/tools/build_project.py b/tools/build_project.py index 6ff69d84..644e06a7 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -103,13 +103,18 @@ def get_git_commit_id(repo: pathlib.Path) -> str: """Get a commit hash for the current git repository.""" def git(*args: str) -> str: - result = subprocess.run( - ["git", "-C", str(repo)] + list(args), - text=True, - capture_output=True, - check=True, - ) - return result.stdout.strip() + try: + result = subprocess.run( + ["git", "-C", str(repo)] + list(args), + text=True, + capture_output=True, + check=True, + ) + return result.stdout.strip() + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f"Git command `git -C {repo} {' '.join(args)}` failed: {exc.stderr.strip()}" + ) from exc # Get the current commit ID commit_id = git("rev-parse", "HEAD")[:8]