test: use earthly-next branch for some tests (#4158)

Adds a new workflow to run the +docker-tests-no-qemu-group* tests using the buildkit version from the earthly-next branch. --------- Signed-off-by: Alex Couture-Beil <[email protected]>
earthly · May 28, 2024 · 2131356 · 2131356
1 parent d3743e7
commit 2131356
Show file tree

Hide file tree

Showing 4 changed files with 236 additions and 5 deletions.
diff --git a/.github/actions/stage2-setup/action.yml b/.github/actions/stage2-setup/action.yml
@@ -18,6 +18,10 @@ inputs:
     required: false
     type: boolean
     default: false
+  USE_NEXT:
+    required: false
+    type: boolean
+    default: false
   SATELLITE_NAME:
     required: false
     type: string
@@ -67,16 +71,26 @@ runs:
       shell: bash
     - run: |-
         echo "Extracting earthly binary from stage1 of build"
-        BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging TAG=${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} ${{inputs.SUDO}} ./earthly upgrade
+        export TAG=${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}}
+        if [ "${{inputs.USE_NEXT}}" = "true" ]; then export TAG="$TAG-ticktock"; fi
+        BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging ${{inputs.SUDO}} ./earthly upgrade
         ${{inputs.SUDO}} chown -R $USER ~/.earthly # restore non-sudo user ownership
         test -n "${{inputs.BUILT_EARTHLY_PATH}}" || (echo "BUILT_EARTHLY_PATH is empty" && exit 1)
         mkdir -p $(dirname "${{inputs.BUILT_EARTHLY_PATH}}")
-        ${{inputs.SUDO}} mv ${HOME}/.earthly/earthly-${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} "${{inputs.BUILT_EARTHLY_PATH}}"
+        ${{inputs.SUDO}} ls "${HOME}/.earthly/"
+        ${{inputs.SUDO}} mv "${HOME}/.earthly/earthly-$TAG" "${{inputs.BUILT_EARTHLY_PATH}}"
         echo "extracted ${{inputs.BUILT_EARTHLY_PATH}}"
       shell: bash
+    - if: ${{ inputs.USE_NEXT == 'true' }}
+      run: |-
+        export expected_buildkit_client_sha="$(cat earthly-next | head -c 12)"
+        test -n "$expected_buildkit_client_sha" || ( echo "expected_buildkit_client_sha is empty" && exit 1)
+        (strings ${{inputs.BUILT_EARTHLY_PATH}} | grep "$expected_buildkit_client_sha" ) || ( echo "expected to find $expected_buildkit_client_sha in earthly binary" && exit 1)
+        echo "correctly found $expected_buildkit_client_sha in earthly binary; this confirms earthly-next was used"
+      shell: bash
     - run: |-
         echo "Setting up mirror credentials in .arg and .secret"
-        export earthly=${{inputs.BUILT_EARTHLY_PATH}}
+        export earthly="${{inputs.BUILT_EARTHLY_PATH}}"
         # setup secrets
         echo "DOCKERHUB_MIRROR_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/user || kill $$)" > .secret
         echo "DOCKERHUB_MIRROR_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/pass || kill $$)" >> .secret

diff --git a/.github/workflows/build-earthly.yml b/.github/workflows/build-earthly.yml
@@ -19,11 +19,15 @@ on:
         required: false
         type: boolean
         default: false
+      USE_NEXT:
+        required: false
+        type: boolean
+        default: false
 
 jobs:
 
   build-earthly:
-    name: build (and push) earthly using ${{inputs.BINARY}}
+    name: build (and push) earthly using ${{inputs.BINARY}} and earthly-next=${{inputs.USE_NEXT}}
     if: ${{!inputs.SKIP_JOB}}
     runs-on: ${{inputs.RUNS_ON}}
     env:
@@ -71,6 +75,9 @@ jobs:
       - name: Podman Login
         run: ${{inputs.SUDO}} ${{inputs.BINARY}} login docker.io --username "${{ secrets.DOCKERHUB_USERNAME }}" --password "${{ secrets.DOCKERHUB_TOKEN }}"
         if: inputs.BINARY == 'podman'
+      - name: Update Buildkit to earthly-next
+        if: inputs.USE_NEXT
+        run: ${{inputs.SUDO}} $(which earthly) +update-buildkit --BUILDKIT_GIT_SHA=$(cat earthly-next)
       - name: Build latest earthly using released earthly
         run: ${{inputs.SUDO}} $(which earthly) --use-inline-cache +for-linux
       - name: Earthly bootstrap using latest earthly build
@@ -81,7 +88,10 @@ jobs:
           EARTHLY_VERSION_FLAG_OVERRIDES="$(tr -d '\n' < .earthly_version_flag_overrides)"
           echo "EARTHLY_VERSION_FLAG_OVERRIDES=$EARTHLY_VERSION_FLAG_OVERRIDES" >> "$GITHUB_ENV"
       - name: Build and push +ci-release using latest earthly build
-        run: ${{inputs.SUDO}} ./build/linux/amd64/earthly --ci --push +ci-release --TAG_SUFFIX="${{inputs.RUNS_ON}}-${{inputs.BINARY}}"
+        run: |-
+          export TAG_SUFFIX="${{inputs.RUNS_ON}}-${{inputs.BINARY}}"
+          if [ "${{inputs.USE_NEXT}}" = "true" ]; then export TAG_SUFFIX="$TAG_SUFFIX-ticktock"; fi
+          ${{inputs.SUDO}} ./build/linux/amd64/earthly --ci --push --build-arg TAG_SUFFIX +ci-release
       - name: Buildkit logs (runs on failure)
         run: ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd
         if: ${{ failure() }}
diff --git a/.github/workflows/ci-earthly-next-docker-ubuntu.yml b/.github/workflows/ci-earthly-next-docker-ubuntu.yml
@@ -0,0 +1,196 @@
+name: Earthly Next Docker CI Ubuntu
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+      - '.github/renovate.json5'
+      - '.github/CODEOWNERS'
+      - 'LICENSE'
+  pull_request:
+    branches: [ main ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-earthly-with-next:
+    uses: ./.github/workflows/build-earthly.yml
+    with:
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group1:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group1"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group2:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group2"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group3:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group3"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group4:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group4"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group5:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group5"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group6:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group6"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group7:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group7"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group8:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group8"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group9:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group9"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group10:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group10"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group11:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group11"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
+
+  earthly-next-docker-tests-no-qemu-group12:
+    needs: build-earthly-with-next
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      TEST_TARGET: "+test-no-qemu-group12"
+      BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly"
+      RUNS_ON: "ubuntu-latest"
+      BINARY: "docker"
+      USE_NEXT: true
+      SUDO: ""
+      SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }}
+      EXTRA_ARGS: "--auto-skip"
+    secrets: inherit
diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml
@@ -38,6 +38,10 @@ on:
         required: false
         type: boolean
         default: false
+      USE_NEXT:
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   test:
@@ -67,6 +71,7 @@ jobs:
           BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}"
           BINARY: "${{ inputs.BINARY }}"
           USE_QEMU: "${{ inputs.USE_QEMU }}"
+          USE_NEXT: "${{ inputs.USE_NEXT }}"
           SUDO: "${{ inputs.SUDO }}"
           USE_SATELLITE: "${{ inputs.USE_SATELLITE }}"
           SATELLITE_NAME: "${{ inputs.SATELLITE_NAME }}"
@@ -78,6 +83,12 @@ jobs:
       - name: Execute ${{ inputs.TEST_TARGET }} (Earthly Only)
         run: |-
           ${{inputs.SUDO}} ${{ inputs.BUILT_EARTHLY_PATH }} ${{inputs.EXTRA_ARGS}} --ci -P ${{inputs.TEST_TARGET}}
+      - name: Display buildkit version
+        run: |-
+          ${{inputs.SUDO}} ${{inputs.BINARY}} ps -a
+          ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd |& grep 'starting earthly-buildkit'
+        shell: bash
+        if: ${{ ! inputs.USE_SATELLITE }}
       - name: Execute fail test
         run: |
           ! ${{inputs.SUDO}} GITHUB_ACTIONS="" ${{ inputs.BUILT_EARTHLY_PATH }} ${{inputs.EXTRA_ARGS}} --ci ./tests/fail+test-fail