release work

AztecProtocol · Jan 29, 2025 · 11460d3 · 11460d3
1 parent beb28fc
commit 11460d3
Show file tree

Hide file tree

Showing 26 changed files with 306 additions and 206 deletions.
diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml
@@ -21,34 +21,37 @@ concurrency:
 jobs:
   ci:
     runs-on: ubuntu-latest
+    # Build for both architectures, but arm is only on master.
+    strategy:
+      matrix:
+        include:
+          - arch: amd64
+          - arch: arm64
+            master_only: true
     if: >
       (github.event.action != 'labeled' && github.event.pull_request.draft == false) ||
-      (github.event.action == 'labeled' && github.event.label.name == 'trigger-workflow')
+      (github.event.action == 'labeled' && github.event.label.name == 'trigger-workflow') &&
+      (matrix.master_only != true || github.ref == 'refs/heads/master')
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          # Ensure it is not an artificial merge commit
+          # Ensure we checkout our actual commit, and not the result of merging the PR to the target.
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
       - name: Setup
         run: |
-          # Ensure we can SSH into the spot instances we request
+          # Ensure we can SSH into the spot instances we request.
           mkdir -p ~/.ssh
           echo ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | base64 --decode > ~/.ssh/build_instance_key
           chmod 600 ~/.ssh/build_instance_key
       - name: Run
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          ARCH: ${{ matrix.arch }}
+          LOG_ID: ${{ github.run_id }}
         run: |
-          export LOG_ID=${{ github.run_id }}
-          export BRANCH=${{ github.event.pull_request.head.ref || github.ref_name }}
-          [ "${{ github.ref_name }}" == "master" ] && export MASTER=1 || export MASTER=0
           ./ci.sh ec2
-      - name: Deploy
-        if: startsWith(github.ref, 'refs/tags/v') && github.ref =~ 'refs/tags/v[0-9]+\\.[0-9]+\\.[0-9]+'
-        run: |
-          ./ci.sh deploy "${GITHUB_REF#refs/tags/}"
       # TODO: We will need to extract the result from the remote container before termination.
       # Also only on master?
       # - name: Store benchmark result

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -1,3 +1,5 @@
+/.devcontainer/ @charlielye
+/.github/ @charlielye
 /aztec-up/ @charlielye
 /release-image/ @charlielye
 /build-images/ @charlielye

diff --git a/aztec-up/bin/aztec-up b/aztec-up/bin/aztec-up
@@ -6,9 +6,9 @@ export NON_INTERACTIVE=1
 INSTALL_URI=${INSTALL_URI:-https://install.aztec.network}
 
 if [ -n "$VERSION" ] && [ "$VERSION" != "latest" ]; then
-  INSTALL_URL="$INSTALL_URI/$VERSION/aztec-install"
+  install_url="$INSTALL_URI/$VERSION/aztec-install"
 else
-  INSTALL_URL="$INSTALL_URI/aztec-install"
+  install_url="$INSTALL_URI/aztec-install"
 fi
 
-bash <(curl -s $INSTALL_URL)
+bash <(curl -s $install_url)
diff --git a/aztec-up/bootstrap.sh b/aztec-up/bootstrap.sh
@@ -20,6 +20,38 @@ function test {
   test_cmds | parallelise
 }
 
+function release {
+  echo_header "aztec-up release"
+  local version=$REF_NAME
+
+  # Function to compare versions.
+  version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }
+
+  # Read the current version from S3.
+  local current_version=$(aws s3 cp s3://install.aztec.network/VERSION - 2>/dev/null || echo "0.0.0")
+
+  # Validate that version is a valid semver.
+  if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "Warning: $version is not a valid semver version. Skipping version comparison."
+  else
+    # Check if new version is greater than current version.
+    if version_gt "$version" "$current_version"; then
+      echo "Uploading new version: $version"
+
+      # Upload new version to root.
+      aws s3 sync ./bin s3://install.aztec.network/
+
+      # Update VERSION file.
+      echo "$version" | aws s3 cp - s3://install.aztec.network/VERSION
+    else
+      echo "New version $version is not greater than current version $current_version. Skipping root upload."
+    fi
+  fi
+
+  # Always create a version directory and upload files there.
+  aws s3 sync ./bin s3://install.aztec.network/$version/
+}
+
 case "$cmd" in
   ""|"full")
     build_dind_image
@@ -30,6 +62,9 @@ case "$cmd" in
   "test")
     test
     ;;
+  "release")
+    release
+    ;;
   *)
     echo "Unknown command: $cmd"
     exit 1

diff --git a/aztec-up/terraform/main.tf b/aztec-up/terraform/main.tf
@@ -69,45 +69,6 @@ resource "aws_s3_bucket_policy" "install_bucket_policy" {
   })
 }
 
-# Upload files to s3 bucket if changes were detected
-resource "null_resource" "upload_public_directory" {
-  triggers = {
-    always_run = "${timestamp()}"
-  }
-
-  provisioner "local-exec" {
-    interpreter = ["/bin/bash", "-c"]
-    command     = <<EOT
-      # Function to compare versions
-      version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; }
-
-      # Read the current version from S3
-      CURRENT_VERSION=$(aws s3 cp s3://${aws_s3_bucket.install_bucket.id}/VERSION - 2>/dev/null || echo "0.0.0")
-
-      # Validate that var.VERSION is a valid semver
-      if [[ ! "${var.VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-        echo "Warning: ${var.VERSION} is not a valid semver version. Skipping version comparison."
-      else
-        # Check if new version is greater than current version
-        if version_gt "${var.VERSION}" "$CURRENT_VERSION"; then
-          echo "Uploading new version ${var.VERSION}"
-
-          # Upload new version to root
-          aws s3 sync ../bin s3://${aws_s3_bucket.install_bucket.id}/
-
-          # Update VERSION file
-          echo "${var.VERSION}" | aws s3 cp - s3://${aws_s3_bucket.install_bucket.id}/VERSION
-        else
-          echo "New version ${var.VERSION} is not greater than current version $CURRENT_VERSION. Skipping root upload."
-        fi
-      fi
-
-      # Always create a version directory and upload files there
-      aws s3 sync ../bin s3://${aws_s3_bucket.install_bucket.id}/${var.VERSION}/
-    EOT
-  }
-}
-
 resource "aws_cloudfront_distribution" "install" {
   origin {
     domain_name = aws_s3_bucket.install_bucket.website_endpoint

diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json
@@ -55,7 +55,8 @@
       "binaryDir": "build-darwin-arm64",
       "cacheVariables": {
         "CMAKE_BUILD_TYPE": "Release",
-        "CMAKE_TOOLCHAIN_FILE": "/opt/osxcross/toolchain.cmake"
+        "CMAKE_TOOLCHAIN_FILE": "/opt/osxcross/toolchain.cmake",
+        "CMAKE_EXE_LINKER_FLAGS": "-fuse-ld=/opt/osxcross/bin/arm64-apple-darwin23-ld"
       },
       "environment": {
         "OSXCROSS_HOST": "arm64-apple-darwin23",
@@ -69,6 +70,9 @@
       "displayName": "Apple amd64 Cross-compile",
       "inherits": "darwin-arm64",
       "binaryDir": "build-darwin-amd64",
+      "cacheVariables": {
+        "CMAKE_EXE_LINKER_FLAGS": "-fuse-ld=/opt/osxcross/bin/x86_64-apple-darwin23-ld"
+      },
       "environment": {
         "OSXCROSS_HOST": "x86_64-apple-darwin23"
       }
@@ -632,7 +636,13 @@
       "configurePreset": "wasm",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": ["barretenberg.wasm", "barretenberg", "wasi", "env"]
+      "targets": [
+        "barretenberg.wasm",
+        "barretenberg.wasm.gz",
+        "barretenberg",
+        "wasi",
+        "env"
+      ]
     },
     {
       "name": "wasm-threads-dbg",
@@ -653,7 +663,7 @@
       "configurePreset": "wasm-threads",
       "inheritConfigureEnvironment": true,
       "jobs": 0,
-      "targets": ["barretenberg.wasm"]
+      "targets": ["barretenberg.wasm", "barretenberg.wasm.gz"]
     },
     {
       "name": "xray",

diff --git a/barretenberg/cpp/bootstrap.sh b/barretenberg/cpp/bootstrap.sh
@@ -2,6 +2,7 @@
 source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
 cmd=${1:-}
+[ -n "$cmd" ] && shift
 
 export preset=clang16-assert
 export pic_preset="clang16-pic"
@@ -17,7 +18,10 @@ function build_native {
     cmake --build --preset $preset
     cache_upload barretenberg-release-$hash.tar.gz build/bin
   fi
+}
 
+function build_world_state_napi {
+  set -eu
   (cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
   if ! cache_download barretenberg-release-world-state-$hash.tar.gz; then
     rm -f build-pic/CMakeCache.txt
@@ -27,30 +31,41 @@ function build_native {
   fi
 }
 
+function build_darwin {
+  local arch=${1:-$(arch)}
+  if ! cache_download barretenberg-darwin-$hash.tar.gz; then
+    # Download sdk.
+    local osx_sdk="MacOSX14.0.sdk"
+    if ! [ -d "/opt/osxcross/SDK/$osx_sdk" ]; then
+      echo "Downloading $osx_sdk..."
+      local osx_sdk_url="https://github.com/joseluisq/macosx-sdks/releases/download/14.0/${osx_sdk}.tar.xz"
+      curl -sSL "$osx_sdk_url" | sudo tar -xJ -C /opt/osxcross/SDK && sudo rm -rf /opt/osxcross/SDK/$osx_sdk/System
+    fi
+
+    rm -f build-darwin-$arch/CMakeCache.txt
+    cmake --preset darwin-$arch && cmake --build --preset darwin-$arch --target bb
+    cache_upload barretenberg-darwin-$hash.tar.gz build-darwin-$arch/bin
+  fi
+}
+
 # Build single threaded wasm. Needed when no shared mem available.
 function build_wasm {
   set -eu
   if ! cache_download barretenberg-wasm-$hash.tar.gz; then
     rm -f build-wasm/CMakeCache.txt
-    cmake --preset wasm
-    cmake --build --preset wasm
-    /opt/wasi-sdk/bin/llvm-strip ./build-wasm/bin/barretenberg.wasm
+    cmake --preset wasm && cmake --build --preset wasm
     cache_upload barretenberg-wasm-$hash.tar.gz build-wasm/bin
   fi
-  (cd ./build-wasm/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
 }
 
 # Build multi-threaded wasm. Requires shared memory.
 function build_wasm_threads {
   set -eu
   if ! cache_download barretenberg-wasm-threads-$hash.tar.gz; then
     rm -f build-wasm-threads/CMakeCache.txt
-    cmake --preset wasm-threads
-    cmake --build --preset wasm-threads
-    /opt/wasi-sdk/bin/llvm-strip ./build-wasm-threads/bin/barretenberg.wasm
+    cmake --preset wasm-threads && cmake --build --preset wasm-threads
     cache_upload barretenberg-wasm-threads-$hash.tar.gz build-wasm-threads/bin
   fi
-  (cd ./build-wasm-threads/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
 }
 
 # Download ignition transcripts. Only needed for tests.
@@ -60,11 +75,25 @@ function download_old_crs {
   cd ./srs_db && ./download_ignition.sh 3 && ./download_grumpkin.sh
 }
 
-export -f build_native build_wasm build_wasm_threads download_old_crs
+function build_release {
+  rm -rf build-release
+  mkdir build-release
+  local arch=$(arch)
+  tar -czf build-release/barretenberg-$arch-linux.tar.gz -C build/bin bb
+  tar -czf build-release/barretenberg-$arch-darwin.tar.gz -C build-darwin-$arch/bin bb
+  tar -czf build-release/barretenberg-wasm.tar.gz -C build-wasm/bin barretenberg.wasm
+  tar -czf build-release/barretenberg-debug-wasm.tar.gz -C build-wasm/bin barretenberg-debug.wasm
+  tar -czf build-release/barretenberg-threads-wasm.tar.gz -C build-wasm-threads/bin barretenberg.wasm
+  tar -czf build-release/barretenberg-threads-debug-wasm.tar.gz -C build-wasm-threads/bin barretenberg-debug.wasm
+}
+
+export -f build_native build_darwin build_world_state_napi build_wasm build_wasm_threads download_old_crs
 
 function build {
   echo_header "bb cpp build"
-  parallel --line-buffered --tag denoise {} ::: build_native build_wasm build_wasm_threads download_old_crs
+  parallel --line-buffered --tag denoise {} ::: \
+    build_native build_world_state_napi build_wasm build_wasm_threads build_darwin download_old_crs
+  build_release
 }
 
 # Print every individual test command. Can be fed into gnu parallel.
@@ -100,7 +129,8 @@ function build_benchmarks {
   fi
 }
 
-function benchmark {
+function bench {
+  echo_header "bb bench"
   build_benchmarks
 
   export HARDWARE_CONCURRENCY=16
@@ -147,6 +177,12 @@ function benchmark {
     > ./bench-out/bench.json
 }
 
+# Upload assets to release.
+function release {
+  echo_header "bb cpp release"
+  gh release upload $REF_NAME build-release/* --clobber
+}
+
 case "$cmd" in
   "clean")
     git clean -fdx
@@ -173,10 +209,10 @@ case "$cmd" in
   "test-cmds")
     test_cmds
     ;;
-  "bench")
-    benchmark
+  bench|release|build_native|build_wasm|build_wasm_threads|build_darwin|build_release)
+    $cmd $@
     ;;
   *)
     echo "Unknown command: $cmd"
     exit 1
-esac
+esac
diff --git a/barretenberg/cpp/cmake/arch.cmake b/barretenberg/cpp/cmake/arch.cmake
@@ -5,7 +5,7 @@ if(WASM)
     add_compile_options(-fno-exceptions -fno-slp-vectorize)
 endif()
 
-if(NOT WASM AND NOT APPLE AND NOT ARM)
+if(NOT WASM AND NOT ARM)
     message(STATUS "Target architecture: ${TARGET_ARCH}")
     add_compile_options(-march=${TARGET_ARCH})
 endif()
diff --git a/barretenberg/cpp/scripts/gzip-wasm.sh b/barretenberg/cpp/scripts/gzip-wasm.sh
@@ -1,3 +1,8 @@
 #!/bin/sh
-(cd ./build-wasm/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
-(cd ./build-wasm-threads/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
+if [ -z "${NO_STRIP:-}" ]; then
+  (cd ./build-wasm/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
+  (cd ./build-wasm-threads/bin && gzip barretenberg.wasm -c > barretenberg.wasm.gz)
+else
+  (cd ./build-wasm/bin && gzip barretenberg-debug.wasm -c > barretenberg.wasm.gz)
+  (cd ./build-wasm-threads/bin && gzip barretenberg-debug.wasm -c > barretenberg.wasm.gz)
+fi
diff --git a/barretenberg/cpp/scripts/strip-wasm.sh b/barretenberg/cpp/scripts/strip-wasm.sh