diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fd5219d69..1714f8073 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.gitlab.com/finestructure/spi-base:1.2.0 +FROM registry.gitlab.com/finestructure/spi-base:1.2.2 # Install SPM build dependencies RUN apt-get update && apt-get install -y curl git make unzip \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a914772b9..7c4b6c6d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,20 +29,43 @@ jobs: name: Test runs-on: ubuntu-latest container: - image: registry.gitlab.com/finestructure/spi-base:1.2.0 + image: registry.gitlab.com/finestructure/spi-base:1.2.2 options: --privileged steps: - name: GH Runner bug workaround run: sysctl -w vm.mmap_rnd_bits=28 + - name: Checkout code uses: actions/checkout@v4 with: { 'fetch-depth': 0 } - - name: Install unzip - run: apt-get update && apt-get install -y unzip + + - name: Install unzip (for tests) and zstd (for faster caching) + run: apt-get update && apt-get install -y unzip zstd + + - name: Restore .build + if: ${{ !(github.run_attempt > 1) }} # Because maybe the cache is causing issues + id: "restore-cache" + uses: actions/cache/restore@v4 + with: + path: .build + key: "spi-debug-build-${{ runner.os }}-${{ github.event.after }}" + restore-keys: "spi-debug-build-${{ runner.os }}-" + + - name: Build tests + run: cp .env.testing.template .env.testing && make build-tests + + - name: Cache .build + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .build + key: "spi-debug-build-${{ runner.os }}-${{ github.event.after }}" + - name: Run tests - run: cp .env.testing.template .env.testing && make test + run: cp .env.testing.template .env.testing && make run-tests env: COLLECTION_SIGNING_PRIVATE_KEY: ${{ secrets.COLLECTION_SIGNING_PRIVATE_KEY }} + services: spi_test_0: image: postgres:16-alpine @@ -137,20 +160,44 @@ jobs: name: Release build runs-on: ubuntu-latest container: - image: registry.gitlab.com/finestructure/spi-base:1.2.0 + image: registry.gitlab.com/finestructure/spi-base:1.2.2 options: --privileged steps: - name: GH Runner bug workaround run: sysctl -w vm.mmap_rnd_bits=28 + - name: Checkout code uses: actions/checkout@v4 with: { 'fetch-depth': 0 } - - name: Test release + + - name: Install zstd (for faster caching) + run: apt-get update && apt-get install -y zstd + + - name: Restore .build + if: ${{ !(github.run_attempt > 1) }} # Because maybe the cache is causing issues + id: "restore-cache" + uses: actions/cache/restore@v4 + with: + path: .build + key: "spi-release-build-${{ runner.os }}-${{ github.event.after }}" + restore-keys: "spi-release-build-${{ runner.os }}-" + + - name: Build release run: | # Since we're only pinging the version api endpoint, no database is needed and # we can just copy the development template to ensure the db env variables are set. cp .env.development.template .env.development - swift build -c release --static-swift-stdlib -Xlinker -ljemalloc + swift build -c release --static-swift-stdlib -Xlinker -ljemalloc --enable-experimental-prebuilts + + - name: Cache .build + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .build + key: "spi-release-build-${{ runner.os }}-${{ github.event.after }}" + + - name: Test release + run: | $(swift build --show-bin-path -c release)/Run serve --port 8080 --hostname 0.0.0.0 & sleep 10 echo Probing api/version... diff --git a/.github/workflows/query-performance.yml b/.github/workflows/query-performance.yml index 70b29c357..d3929e449 100644 --- a/.github/workflows/query-performance.yml +++ b/.github/workflows/query-performance.yml @@ -28,13 +28,34 @@ jobs: runs-on: ubuntu-latest continue-on-error: true container: - image: registry.gitlab.com/finestructure/spi-base:1.2.0 + image: registry.gitlab.com/finestructure/spi-base:1.2.2 steps: - name: Checkout code uses: actions/checkout@v4 with: { 'fetch-depth': 0 } - - name: Install unzip - run: apt-get update && apt-get install -y unzip + + - name: Install unzip (for tests) and zstd (for faster caching) + run: apt-get update && apt-get install -y unzip zstd + + - name: Restore .build + if: ${{ !(github.run_attempt > 1) }} # Because maybe the cache is causing issues + id: "restore-cache" + uses: actions/cache/restore@v4 + with: + path: .build + key: "spi-query-performance-build-${{ runner.os }}-${{ github.event.after }}" + restore-keys: "spi-query-performance-build-${{ runner.os }}-" + + - name: Build tests + run: cp .env.testing.template .env.testing && make build-tests + + - name: Cache .build + if: steps.restore-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .build + key: "spi-query-performance-build-${{ runner.os }}-${{ github.event.after }}" + - name: Run tests run: | echo " @@ -45,4 +66,4 @@ jobs: DATABASE_USE_TLS=true DATABASE_USERNAME=spi_dev " >> .env.staging - make test-query-performance + make run-query-performance-tests diff --git a/Dockerfile b/Dockerfile index c90ff28dc..1bbfdc321 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ # ================================ # Build image # ================================ -FROM registry.gitlab.com/finestructure/spi-base:1.2.0 as build +FROM registry.gitlab.com/finestructure/spi-base:1.2.2 as build # Set up a build area WORKDIR /build @@ -37,6 +37,7 @@ COPY . . # Build everything, with optimizations, with static linking, and using jemalloc # N.B.: The static version of jemalloc is incompatible with the static Swift runtime. RUN swift build -c release \ + --enable-experimental-prebuilts \ --static-swift-stdlib \ -Xlinker -ljemalloc @@ -61,7 +62,7 @@ RUN [ -d /build/Resources ] && { mv /build/Resources ./Resources && chmod -R a-w # ================================ # Run image # ================================ -FROM registry.gitlab.com/finestructure/spi-base:1.2.0 +FROM registry.gitlab.com/finestructure/spi-base:1.2.2 # NB sas 2022-09-23: We're not using a dedicated `vapor` user to run the executable, because it # makes managing the data in the checkouts volume difficult. See diff --git a/LOCAL_DEVELOPMENT_SETUP.md b/LOCAL_DEVELOPMENT_SETUP.md index 7c2541439..3387a4d58 100644 --- a/LOCAL_DEVELOPMENT_SETUP.md +++ b/LOCAL_DEVELOPMENT_SETUP.md @@ -238,7 +238,7 @@ The trickiest part of this is to ensure the test or app container can connect to So, in order to run the tests in a Linux container run: ``` -docker run --rm -v "$PWD":/host -w /host --add-host=host.docker.internal:host-gateway registry.gitlab.com/finestructure/spi-base:1.2.0 swift test +docker run --rm -v "$PWD":/host -w /host --add-host=host.docker.internal:host-gateway registry.gitlab.com/finestructure/spi-base:1.2.2 swift test ``` Make sure you use the most recent `spi-base` image. You can find the latest image name in the `test-docker` target, which also provides a convenient way to run all all tests in a docker container. diff --git a/Makefile b/Makefile index 5304330dd..0d5c210b8 100644 --- a/Makefile +++ b/Makefile @@ -28,24 +28,48 @@ else endif build: - swift build --disable-automatic-resolution + swift build --disable-automatic-resolution --enable-experimental-prebuilts run: swift run -test: xcbeautify +build-tests: xcbeautify set -o pipefail \ - && swift test --disable-automatic-resolution \ - 2>&1 | ./xcbeautify --renderer github-actions + && swift build --build-tests \ + --disable-automatic-resolution \ + --enable-experimental-prebuilts \ + 2>&1 | xcbeautify --renderer github-actions -test-query-performance: xcbeautify +run-tests: xcbeautify + set -o pipefail \ + && swift test --skip-build \ + --disable-automatic-resolution \ + --enable-experimental-prebuilts \ + 2>&1 | xcbeautify --renderer github-actions + +test: build-tests run-tests + +build-query-performance-tests: + set -o pipefail \ + && env RUN_QUERY_PERFORMANCE_TESTS=true \ + swift build --build-tests \ + --disable-automatic-resolution \ + --enable-experimental-prebuilts \ + --filter QueryPerformanceTests \ + 2>&1 | xcbeautify --renderer github-actions + +run-query-performance-tests: set -o pipefail \ && env RUN_QUERY_PERFORMANCE_TESTS=true \ - swift test --disable-automatic-resolution \ + swift test --skip-build \ + --disable-automatic-resolution \ + --enable-experimental-prebuilts \ --filter QueryPerformanceTests \ 2>&1 | tee test.log grep "ℹ️" test.log - grep -v "\] Compiling" test.log | ./xcbeautify --renderer github-actions + grep -v "\] Compiling" test.log | xcbeautify --renderer github-actions + +test-query-performance: build-query-performance-tests run-query-performance-tests test-fast: @echo Skipping image snapshot tests @@ -53,10 +77,7 @@ test-fast: swift test --disable-automatic-resolution xcbeautify: - rm -rf .build/checkouts/xcbeautify - git clone https://github.com/cpisciotta/xcbeautify.git .build/checkouts/xcbeautify - cd .build/checkouts/xcbeautify && git checkout 2.25.1 && make build - binpath=`cd .build/checkouts/xcbeautify && swift build -c release --show-bin-path` && ln -sf $$binpath/xcbeautify + command -v xcbeautify docker-build: version docker build -t $(DOCKER_IMAGE):$(VERSION) . @@ -68,7 +89,7 @@ test-docker: @# run tests inside a docker container docker run --rm -v "$(PWD)":/host -w /host \ --add-host=host.docker.internal:host-gateway \ - registry.gitlab.com/finestructure/spi-base:1.2.0 \ + registry.gitlab.com/finestructure/spi-base:1.2.2 \ make test test-e2e: db-reset reconcile ingest analyze diff --git a/Tests/AppTests/QueryPerformanceTests.swift b/Tests/AppTests/QueryPerformanceTests.swift index 9d3d5d06e..ec1127a1c 100644 --- a/Tests/AppTests/QueryPerformanceTests.swift +++ b/Tests/AppTests/QueryPerformanceTests.swift @@ -180,7 +180,7 @@ struct QueryPerformanceTests { JOIN versions v ON v.package_id = p.id WHERE v.reference ->> 'branch' = r.default_branch """) - try await assertQueryPerformance(query, expectedCost: 150_000, variation: 5000) + try await assertQueryPerformance(query, expectedCost: 160_000, variation: 5000) } }