break up ast tests to prevent OOM errors (#3974)

additionally add a retry attempt in the buildkite test, to avoid having to re-run all groups if a single group fails. Signed-off-by: Alex Couture-Beil <[email protected]>
earthly · Apr 2, 2024 · e46366d · e46366d
1 parent 1755046
commit e46366d
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 14 deletions.
diff --git a/.buildkite/tests.sh b/.buildkite/tests.sh
@@ -103,11 +103,14 @@ set -x
 # stop the released earthly buildkitd container (to preserve memory)
 docker rm -f earthly-buildkitd 2> /dev/null || true
 
-echo "Execute tests"
+max_attempts=2
 for target in \
         +test-misc-group1 \
         +test-misc-group2 \
         +test-misc-group3 \
+        +test-ast-group1 \
+        +test-ast-group2 \
+        +test-ast-group3 \
         +test-no-qemu-group1 \
         +test-no-qemu-group2 \
         +test-no-qemu-group3 \
@@ -119,12 +122,28 @@ for target in \
         +test-no-qemu-slow \
         +test-qemu \
         ; do
-    echo "=== running $target ==="
-    "$earthly" --ci -P --exec-stats-summary=- "$target" || ( echo "$target failed" && exit 1 )
-    # kill buildkitd to release memory (the macstadium machines have limited memory)
-    docker rm -f earthly-dev-buildkitd 2> /dev/null || true
+    for attempt in $(seq 1 "$max_attempts"); do
+        # kill buildkitd to release memory (the macstadium machines have limited memory)
+        docker rm -f earthly-dev-buildkitd 2> /dev/null || true
+
+        echo "=== running $target (attempt $attempt/$max_attempts ==="
+        set +e
+        "$earthly" --ci -P --exec-stats-summary=- "$target"
+        exit_code="$?"
+        set -e
+
+        if [ "$exit_code" = "0" ]; then
+            echo "$target passed"
+            break
+        fi
+
+        echo "$target failed"
+        if [ "$attempt" = "$max_attempts" ]; then
+            echo "final attempt reached, giving up"
+            exit 1
+        fi
+    done
 done
 
-
 echo "Execute fail test"
 bash -c "! $earthly --ci ./tests/fail+test-fail"
diff --git a/Earthfile b/Earthfile
@@ -658,6 +658,7 @@ test-misc:
     BUILD +test-misc-group1
     BUILD +test-misc-group2
     BUILD +test-misc-group3
+    BUILD +test-ast
 
 test-misc-group1:
     BUILD +unit-test
@@ -667,7 +668,20 @@ test-misc-group2:
 
 test-misc-group3:
     BUILD +earthly-script-no-stdout
-    BUILD --pass-args ./ast/tests+all
+
+test-ast:
+    BUILD +test-ast-group1
+    BUILD +test-ast-group2
+    BUILD +test-ast-group3
+
+test-ast-group1:
+    BUILD --pass-args ./ast/tests+group1
+
+test-ast-group2:
+    BUILD --pass-args ./ast/tests+group2
+
+test-ast-group3:
+    BUILD --pass-args ./ast/tests+group3
 
 # test-no-qemu-group1 runs the tests from ./tests+ga-no-qemu-group1
 test-no-qemu-group1:

diff --git a/ast/tests/Earthfile b/ast/tests/Earthfile
@@ -7,6 +7,11 @@ COPY ../..+earthly/earthly /usr/local/bin/earthly
 
 # all runs all tests
 all:
+    BUILD +group1
+    BUILD +group2
+    BUILD +group3
+
+group1:
     BUILD +test \
         --TEST=build-arg \
         --TEST=builtin-args \
@@ -29,7 +34,10 @@ all:
         --TEST=file-copying \
         --TEST=from-expose \
         --TEST=gen-dockerfile \
-        --TEST=git-clone \
+        --TEST=git-clone
+
+group2:
+    BUILD +test \
         --TEST=host \
         --TEST=host-bind \
         --TEST=if-exists \
@@ -52,12 +60,15 @@ all:
         --TEST=with-docker-compose \
         --TEST=command \
         --TEST=import
-    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" --TEST=args
-    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" --TEST=if
-    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" --TEST=for
-    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" --TEST=empty-targets
-    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" --TEST=quotes
-    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" --TEST=documentation
+
+group3:
+    BUILD +test --INPUT_FROM_DIR=./ --INPUT_FROM_TARGET="" \
+        --TEST=args \
+        --TEST=if \
+        --TEST=for \
+        --TEST=empty-targets \
+        --TEST=quotes \
+        --TEST=documentation
 
 # update-all updates all the expected values of the tests
 update-all: