break up tests for mac+buildkite tests (#3954)

The mac instances that run the buildkite tests only have 8GB (7GB under
the docker VM). This breaks up the tests into more granular groups to
isolate memory-intensive tests.

.buildkite/tests.sh

@@ -53,6 +53,15 @@ if [ -n "$download_url" ]; then
     released_earthly=./earthly-released
 fi
 
+echo "docker login"
+set +x # dont echo secrets
+DOCKER_USER="$("$released_earthly" secret --org earthly-technologies --project core get -n dockerhub/user)"
+DOCKER_TOKEN="$("$released_earthly" secret --org earthly-technologies --project core get -n dockerhub/token)"
+test -n "$DOCKER_USER" || (echo "failed to get DOCKER_USER" && exit 1)
+test -n "$DOCKER_TOKEN" || (echo "failed to get DOCKER_TOKEN" && exit 1)
+echo "$DOCKER_TOKEN" | docker login --username "$DOCKER_USER" --password-stdin
+set -x
+
 echo "Prune cache for cross-version compatibility"
 "$released_earthly" prune --reset
 
@@ -72,6 +81,8 @@ do
     sleep $(( att_num++ ))
 done
 
+"$earthly" config global.buildkit_max_parallelism 2
+
 # Yes, there is a bug in the upstream YAML parser. Sorry about the jank here.
 # https://github.com/go-yaml/yaml/issues/423
 "$earthly" config global.buildkit_additional_config "'[registry.\"docker.io\"]
@@ -80,15 +91,40 @@ done
 
 # setup secrets
 set +x # dont echo secrets
+echo "DOCKERHUB_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub/user || kill $$)" > .secret
+echo "DOCKERHUB_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub/pass || kill $$)" >> .secret
 echo "DOCKERHUB_MIRROR_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/user || kill $$)" > .secret
 echo "DOCKERHUB_MIRROR_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/pass || kill $$)" >> .secret
 # setup args
 echo "DOCKERHUB_MIRROR_AUTH=true" > .arg
 echo "DOCKERHUB_MIRROR=registry-1.docker.io.mirror.corp.earthly.dev" >> .arg
 set -x
 
+# stop the released earthly buildkitd container (to preserve memory)
+docker rm -f earthly-buildkitd 2> /dev/null || true
+
 echo "Execute tests"
-"$earthly" --ci -P +test
+for target in \
+        +test-misc-group1 \
+        +test-misc-group2 \
+        +test-misc-group3 \
+        +test-no-qemu-group1 \
+        +test-no-qemu-group2 \
+        +test-no-qemu-group3 \
+        +test-no-qemu-group4 \
+        +test-no-qemu-group5 \
+        +test-no-qemu-group6 \
+        +test-no-qemu-group7 \
+        +test-no-qemu-group8 \
+        +test-no-qemu-slow \
+        +test-qemu \
+        ; do
+    echo "=== running $target ==="
+    "$earthly" --ci -P --exec-stats-summary=- "$target" || ( echo "$target failed" && exit 1 )
+    # kill buildkitd to release memory (the macstadium machines have limited memory)
+    docker rm -f earthly-dev-buildkitd 2> /dev/null || true
+done
+
 
 echo "Execute fail test"
 bash -c "! $earthly --ci ./tests/fail+test-fail"

.github/workflows/ci-docker-satellites.yml

@@ -50,7 +50,7 @@ jobs:
     needs: build-earthly
     uses: ./.github/workflows/reusable-test.yml
     with:
-      TEST_TARGET: "+test-quick"
+      TEST_TARGET: "+test-misc"
       BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
       RUNS_ON: "ubuntu-latest"
       BINARY: "docker"

.github/workflows/ci-docker-ubuntu.yml

@@ -58,7 +58,7 @@ jobs:
     if: ${{ !failure() }}
     uses: ./.github/workflows/reusable-test.yml
     with:
-      TEST_TARGET: "+test-quick"
+      TEST_TARGET: "+test-misc"
       BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
       RUNS_ON: "ubuntu-latest"
       BINARY: "docker"
@@ -502,7 +502,7 @@ jobs:
       BINARY: "docker"
       SUDO: ""
       SKIP_JOB: ${{ needs.build-earthly.result != 'success' }}
-      TARGET_NAME: "+test-quick"
+      TARGET_NAME: "+test-misc"
       EXTRA_ARGS: "--auto-skip"
     secrets: inherit
 
@@ -608,7 +608,7 @@ jobs:
     uses: ./.github/workflows/reusable-race-test.yml
     with:
       BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
-      TEST_TARGET: "+test-quick"
+      TEST_TARGET: "+test-misc"
       RUNS_ON: "ubuntu-latest"
       USE_QEMU: false
       BINARY: "docker"

.github/workflows/ci-podman-ubuntu.yml

@@ -58,7 +58,7 @@ jobs:
     if: ${{ !failure() }}
     uses: ./.github/workflows/reusable-test.yml
     with:
-      TEST_TARGET: "+test-quick"
+      TEST_TARGET: "+test-misc"
       BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
       RUNS_ON: "ubuntu-latest"
       BINARY: "podman"

Earthfile

@@ -267,7 +267,8 @@ unit-test:
 # offline-test runs offline tests with network set to none
 offline-test:
     FROM +code
-    RUN --network=none go test -run TestOffline ./...
+    RUN --network=none (go test -run TestOffline ./cloud || kill $$) | tee test.log
+    RUN if grep 'no tests to run' test.log; then echo "error: no test found" && exit 1; fi
 
 # submodule-decouple-check checks that go submodules within earthly do not
 # depend on the core earthly project.
@@ -716,12 +717,25 @@ test-no-qemu:
     BUILD --pass-args +test-no-qemu-group2
     BUILD --pass-args +test-no-qemu-group3
     BUILD --pass-args +test-no-qemu-group4
+    BUILD --pass-args +test-no-qemu-group5
+    BUILD --pass-args +test-no-qemu-group6
+    BUILD --pass-args +test-no-qemu-group7
+    BUILD --pass-args +test-no-qemu-group8
     BUILD --pass-args +test-no-qemu-slow
 
-# test-quick runs the unit, offline, and go tests and ensures the earthly script does not write to stdout
-test-quick:
+# test-misc runs misc (non earthly-in-earthly) tests
+test-misc:
+    BUILD +test-misc-group1
+    BUILD +test-misc-group2
+    BUILD +test-misc-group3
+
+test-misc-group1:
     BUILD +unit-test
+
+test-misc-group2:
     BUILD +offline-test
+
+test-misc-group3:
     BUILD +earthly-script-no-stdout
     BUILD --pass-args ./ast/tests+all
 
@@ -745,6 +759,26 @@ test-no-qemu-group4:
     BUILD --pass-args ./tests+ga-no-qemu-group4 \
         --GLOBAL_WAIT_END="$GLOBAL_WAIT_END"
 
+# test-no-qemu-group5 runs the tests from ./tests+ga-no-qemu-group5
+test-no-qemu-group5:
+    BUILD --pass-args ./tests+ga-no-qemu-group5 \
+        --GLOBAL_WAIT_END="$GLOBAL_WAIT_END"
+
+# test-no-qemu-group6 runs the tests from ./tests+ga-no-qemu-group6
+test-no-qemu-group6:
+    BUILD --pass-args ./tests+ga-no-qemu-group6 \
+        --GLOBAL_WAIT_END="$GLOBAL_WAIT_END"
+
+# test-no-qemu-group7 runs the tests from ./tests+ga-no-qemu-group7
+test-no-qemu-group7:
+    BUILD --pass-args ./tests+ga-no-qemu-group7 \
+        --GLOBAL_WAIT_END="$GLOBAL_WAIT_END"
+
+# test-no-qemu-group8 runs the tests from ./tests+ga-no-qemu-group8
+test-no-qemu-group8:
+    BUILD --pass-args ./tests+ga-no-qemu-group8 \
+        --GLOBAL_WAIT_END="$GLOBAL_WAIT_END"
+
 # test-no-qemu-slow runs the tests from ./tests+ga-no-qemu-slow
 test-no-qemu-slow:
     BUILD --pass-args ./tests+ga-no-qemu-slow \

tests/Earthfile

@@ -27,6 +27,8 @@ ga-no-qemu-group1:
     BUILD --pass-args ./import+build
     BUILD --pass-args ./import+build-imported
     BUILD +privileged-test
+
+ga-no-qemu-group2:
     BUILD +copy-test
     BUILD +copy-test-verbose-output
     BUILD +copy-tilde-test
@@ -39,6 +41,8 @@ ga-no-qemu-group1:
     BUILD +builtin-args-invalid-pass-test
     BUILD +builtin-args-cli-tests
     BUILD +parser-smoke-test
+
+ga-no-qemu-group3:
     BUILD --pass-args ./invalid+test-all
     BUILD +excludes-test
     BUILD +secrets-test
@@ -56,6 +60,8 @@ ga-no-qemu-group1:
     BUILD +transitive-args-test
     BUILD +transitive-args-test2
     BUILD +non-transitive-args-test
+
+ga-no-qemu-group4:
     BUILD +star-test
     BUILD +dockerfile-test
     BUILD +fail-test
@@ -81,8 +87,6 @@ ga-no-qemu-group1:
     BUILD +wildcard-build-remote
     BUILD +wildcard-build-pwd
     BUILD +wildcard-build-auto-skip
-
-ga-no-qemu-group2:
     BUILD +target-first-line
     BUILD +absolute-reference-with-relative
     BUILD +end-comment
@@ -92,6 +96,8 @@ ga-no-qemu-group2:
     BUILD +save-artifact-dir-as-dot
     BUILD +save-artifact-force-overwrite
     BUILD +save-remote-artifact-selective
+
+ga-no-qemu-group5:
     BUILD +push-build
     BUILD +build-arg-repeat
     BUILD +arg-scope-requires-shellout-anywhere
@@ -158,7 +164,7 @@ ga-no-qemu-group2:
         BUILD --pass-args ./with-docker-via-command+test
     END
 
-ga-no-qemu-group3:
+ga-no-qemu-group6:
     BUILD +cache-test
     BUILD --pass-args ./do-not-track+test-all
     BUILD --pass-args ./scrub-https-credentials+all
@@ -173,7 +179,7 @@ ga-no-qemu-group3:
     BUILD --pass-args ./with-docker-expose+all
     BUILD --pass-args ./logbus+test-all
 
-ga-no-qemu-group4:
+ga-no-qemu-group7:
     BUILD +cache-mount-arg
     BUILD +infinite-recursion
     BUILD +save-artifact-selective
@@ -187,6 +193,8 @@ ga-no-qemu-group4:
     BUILD --pass-args ./with-docker-compose+all
     BUILD +dotenv-test
     BUILD +allow-privileged-test
+
+ga-no-qemu-group8:
     BUILD --pass-args ./with-docker-registry+all
     BUILD +test-ssh-command-config-via-remote-target
     BUILD +test-ssh-command-config-via-git-clone-earthfile-command
@@ -216,6 +224,10 @@ ga-no-qemu:
     BUILD +ga-no-qemu-group2
     BUILD +ga-no-qemu-group3
     BUILD +ga-no-qemu-group4
+    BUILD +ga-no-qemu-group5
+    BUILD +ga-no-qemu-group6
+    BUILD +ga-no-qemu-group7
+    BUILD +ga-no-qemu-group8
     BUILD +ga-no-qemu-slow
 
 # Note that this target is split up under github action workflows