From c502950b56e9247c659a593567e2a3ff30f08054 Mon Sep 17 00:00:00 2001 From: Alex Couture-Beil Date: Mon, 1 Apr 2024 09:14:38 -0700 Subject: [PATCH] break up tests for mac+buildkite tests (#3954) The mac instances that run the buildkite tests only have 8GB (7GB under the docker VM). This breaks up the tests into more granular groups to isolate memory-intensive tests. --- .buildkite/tests.sh | 38 +++++++++++++++++++- .github/workflows/ci-docker-satellites.yml | 2 +- .github/workflows/ci-docker-ubuntu.yml | 6 ++-- .github/workflows/ci-podman-ubuntu.yml | 2 +- Earthfile | 40 ++++++++++++++++++++-- tests/Earthfile | 20 ++++++++--- 6 files changed, 95 insertions(+), 13 deletions(-) diff --git a/.buildkite/tests.sh b/.buildkite/tests.sh index 7e42682a..cd506184 100755 --- a/.buildkite/tests.sh +++ b/.buildkite/tests.sh @@ -53,6 +53,15 @@ if [ -n "$download_url" ]; then released_earthly=./earthly-released fi +echo "docker login" +set +x # dont echo secrets +DOCKER_USER="$("$released_earthly" secret --org earthly-technologies --project core get -n dockerhub/user)" +DOCKER_TOKEN="$("$released_earthly" secret --org earthly-technologies --project core get -n dockerhub/token)" +test -n "$DOCKER_USER" || (echo "failed to get DOCKER_USER" && exit 1) +test -n "$DOCKER_TOKEN" || (echo "failed to get DOCKER_TOKEN" && exit 1) +echo "$DOCKER_TOKEN" | docker login --username "$DOCKER_USER" --password-stdin +set -x + echo "Prune cache for cross-version compatibility" "$released_earthly" prune --reset @@ -72,6 +81,8 @@ do sleep $(( att_num++ )) done +"$earthly" config global.buildkit_max_parallelism 2 + # Yes, there is a bug in the upstream YAML parser. Sorry about the jank here. # https://github.com/go-yaml/yaml/issues/423 "$earthly" config global.buildkit_additional_config "'[registry.\"docker.io\"] @@ -80,6 +91,8 @@ done # setup secrets set +x # dont echo secrets +echo "DOCKERHUB_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub/user || kill $$)" > .secret +echo "DOCKERHUB_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub/pass || kill $$)" >> .secret echo "DOCKERHUB_MIRROR_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/user || kill $$)" > .secret echo "DOCKERHUB_MIRROR_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/pass || kill $$)" >> .secret # setup args @@ -87,8 +100,31 @@ echo "DOCKERHUB_MIRROR_AUTH=true" > .arg echo "DOCKERHUB_MIRROR=registry-1.docker.io.mirror.corp.earthly.dev" >> .arg set -x +# stop the released earthly buildkitd container (to preserve memory) +docker rm -f earthly-buildkitd 2> /dev/null || true + echo "Execute tests" -"$earthly" --ci -P +test +for target in \ + +test-misc-group1 \ + +test-misc-group2 \ + +test-misc-group3 \ + +test-no-qemu-group1 \ + +test-no-qemu-group2 \ + +test-no-qemu-group3 \ + +test-no-qemu-group4 \ + +test-no-qemu-group5 \ + +test-no-qemu-group6 \ + +test-no-qemu-group7 \ + +test-no-qemu-group8 \ + +test-no-qemu-slow \ + +test-qemu \ + ; do + echo "=== running $target ===" + "$earthly" --ci -P --exec-stats-summary=- "$target" || ( echo "$target failed" && exit 1 ) + # kill buildkitd to release memory (the macstadium machines have limited memory) + docker rm -f earthly-dev-buildkitd 2> /dev/null || true +done + echo "Execute fail test" bash -c "! $earthly --ci ./tests/fail+test-fail" diff --git a/.github/workflows/ci-docker-satellites.yml b/.github/workflows/ci-docker-satellites.yml index 5e23e175..6309254e 100644 --- a/.github/workflows/ci-docker-satellites.yml +++ b/.github/workflows/ci-docker-satellites.yml @@ -50,7 +50,7 @@ jobs: needs: build-earthly uses: ./.github/workflows/reusable-test.yml with: - TEST_TARGET: "+test-quick" + TEST_TARGET: "+test-misc" BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" RUNS_ON: "ubuntu-latest" BINARY: "docker" diff --git a/.github/workflows/ci-docker-ubuntu.yml b/.github/workflows/ci-docker-ubuntu.yml index 78356adb..d9e2e93d 100644 --- a/.github/workflows/ci-docker-ubuntu.yml +++ b/.github/workflows/ci-docker-ubuntu.yml @@ -58,7 +58,7 @@ jobs: if: ${{ !failure() }} uses: ./.github/workflows/reusable-test.yml with: - TEST_TARGET: "+test-quick" + TEST_TARGET: "+test-misc" BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" RUNS_ON: "ubuntu-latest" BINARY: "docker" @@ -502,7 +502,7 @@ jobs: BINARY: "docker" SUDO: "" SKIP_JOB: ${{ needs.build-earthly.result != 'success' }} - TARGET_NAME: "+test-quick" + TARGET_NAME: "+test-misc" EXTRA_ARGS: "--auto-skip" secrets: inherit @@ -608,7 +608,7 @@ jobs: uses: ./.github/workflows/reusable-race-test.yml with: BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" - TEST_TARGET: "+test-quick" + TEST_TARGET: "+test-misc" RUNS_ON: "ubuntu-latest" USE_QEMU: false BINARY: "docker" diff --git a/.github/workflows/ci-podman-ubuntu.yml b/.github/workflows/ci-podman-ubuntu.yml index 4439b998..d4ded8c2 100644 --- a/.github/workflows/ci-podman-ubuntu.yml +++ b/.github/workflows/ci-podman-ubuntu.yml @@ -58,7 +58,7 @@ jobs: if: ${{ !failure() }} uses: ./.github/workflows/reusable-test.yml with: - TEST_TARGET: "+test-quick" + TEST_TARGET: "+test-misc" BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" RUNS_ON: "ubuntu-latest" BINARY: "podman" diff --git a/Earthfile b/Earthfile index 8358d7e8..35be83ee 100644 --- a/Earthfile +++ b/Earthfile @@ -267,7 +267,8 @@ unit-test: # offline-test runs offline tests with network set to none offline-test: FROM +code - RUN --network=none go test -run TestOffline ./... + RUN --network=none (go test -run TestOffline ./cloud || kill $$) | tee test.log + RUN if grep 'no tests to run' test.log; then echo "error: no test found" && exit 1; fi # submodule-decouple-check checks that go submodules within earthly do not # depend on the core earthly project. @@ -716,12 +717,25 @@ test-no-qemu: BUILD --pass-args +test-no-qemu-group2 BUILD --pass-args +test-no-qemu-group3 BUILD --pass-args +test-no-qemu-group4 + BUILD --pass-args +test-no-qemu-group5 + BUILD --pass-args +test-no-qemu-group6 + BUILD --pass-args +test-no-qemu-group7 + BUILD --pass-args +test-no-qemu-group8 BUILD --pass-args +test-no-qemu-slow -# test-quick runs the unit, offline, and go tests and ensures the earthly script does not write to stdout -test-quick: +# test-misc runs misc (non earthly-in-earthly) tests +test-misc: + BUILD +test-misc-group1 + BUILD +test-misc-group2 + BUILD +test-misc-group3 + +test-misc-group1: BUILD +unit-test + +test-misc-group2: BUILD +offline-test + +test-misc-group3: BUILD +earthly-script-no-stdout BUILD --pass-args ./ast/tests+all @@ -745,6 +759,26 @@ test-no-qemu-group4: BUILD --pass-args ./tests+ga-no-qemu-group4 \ --GLOBAL_WAIT_END="$GLOBAL_WAIT_END" +# test-no-qemu-group5 runs the tests from ./tests+ga-no-qemu-group5 +test-no-qemu-group5: + BUILD --pass-args ./tests+ga-no-qemu-group5 \ + --GLOBAL_WAIT_END="$GLOBAL_WAIT_END" + +# test-no-qemu-group6 runs the tests from ./tests+ga-no-qemu-group6 +test-no-qemu-group6: + BUILD --pass-args ./tests+ga-no-qemu-group6 \ + --GLOBAL_WAIT_END="$GLOBAL_WAIT_END" + +# test-no-qemu-group7 runs the tests from ./tests+ga-no-qemu-group7 +test-no-qemu-group7: + BUILD --pass-args ./tests+ga-no-qemu-group7 \ + --GLOBAL_WAIT_END="$GLOBAL_WAIT_END" + +# test-no-qemu-group8 runs the tests from ./tests+ga-no-qemu-group8 +test-no-qemu-group8: + BUILD --pass-args ./tests+ga-no-qemu-group8 \ + --GLOBAL_WAIT_END="$GLOBAL_WAIT_END" + # test-no-qemu-slow runs the tests from ./tests+ga-no-qemu-slow test-no-qemu-slow: BUILD --pass-args ./tests+ga-no-qemu-slow \ diff --git a/tests/Earthfile b/tests/Earthfile index 40f5771a..01c0b3f4 100644 --- a/tests/Earthfile +++ b/tests/Earthfile @@ -27,6 +27,8 @@ ga-no-qemu-group1: BUILD --pass-args ./import+build BUILD --pass-args ./import+build-imported BUILD +privileged-test + +ga-no-qemu-group2: BUILD +copy-test BUILD +copy-test-verbose-output BUILD +copy-tilde-test @@ -39,6 +41,8 @@ ga-no-qemu-group1: BUILD +builtin-args-invalid-pass-test BUILD +builtin-args-cli-tests BUILD +parser-smoke-test + +ga-no-qemu-group3: BUILD --pass-args ./invalid+test-all BUILD +excludes-test BUILD +secrets-test @@ -56,6 +60,8 @@ ga-no-qemu-group1: BUILD +transitive-args-test BUILD +transitive-args-test2 BUILD +non-transitive-args-test + +ga-no-qemu-group4: BUILD +star-test BUILD +dockerfile-test BUILD +fail-test @@ -81,8 +87,6 @@ ga-no-qemu-group1: BUILD +wildcard-build-remote BUILD +wildcard-build-pwd BUILD +wildcard-build-auto-skip - -ga-no-qemu-group2: BUILD +target-first-line BUILD +absolute-reference-with-relative BUILD +end-comment @@ -92,6 +96,8 @@ ga-no-qemu-group2: BUILD +save-artifact-dir-as-dot BUILD +save-artifact-force-overwrite BUILD +save-remote-artifact-selective + +ga-no-qemu-group5: BUILD +push-build BUILD +build-arg-repeat BUILD +arg-scope-requires-shellout-anywhere @@ -158,7 +164,7 @@ ga-no-qemu-group2: BUILD --pass-args ./with-docker-via-command+test END -ga-no-qemu-group3: +ga-no-qemu-group6: BUILD +cache-test BUILD --pass-args ./do-not-track+test-all BUILD --pass-args ./scrub-https-credentials+all @@ -173,7 +179,7 @@ ga-no-qemu-group3: BUILD --pass-args ./with-docker-expose+all BUILD --pass-args ./logbus+test-all -ga-no-qemu-group4: +ga-no-qemu-group7: BUILD +cache-mount-arg BUILD +infinite-recursion BUILD +save-artifact-selective @@ -187,6 +193,8 @@ ga-no-qemu-group4: BUILD --pass-args ./with-docker-compose+all BUILD +dotenv-test BUILD +allow-privileged-test + +ga-no-qemu-group8: BUILD --pass-args ./with-docker-registry+all BUILD +test-ssh-command-config-via-remote-target BUILD +test-ssh-command-config-via-git-clone-earthfile-command @@ -216,6 +224,10 @@ ga-no-qemu: BUILD +ga-no-qemu-group2 BUILD +ga-no-qemu-group3 BUILD +ga-no-qemu-group4 + BUILD +ga-no-qemu-group5 + BUILD +ga-no-qemu-group6 + BUILD +ga-no-qemu-group7 + BUILD +ga-no-qemu-group8 BUILD +ga-no-qemu-slow # Note that this target is split up under github action workflows