Skip to content

Commit

Permalink
feat(ci): don't build storage on compute-releases and vice versa
Browse files Browse the repository at this point in the history
  • Loading branch information
jcgruenhage committed Feb 19, 2025
1 parent 86993b9 commit f95ede4
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 8 deletions.
52 changes: 46 additions & 6 deletions .github/workflows/build_and_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,7 @@ jobs:

neon-image-arch:
needs: [ check-permissions, build-build-tools-image, tag ]
if: "!(github.ref_name == 'release-compute' || github.base_ref == 'release-compute')"
strategy:
matrix:
arch: [ x64, arm64 ]
Expand Down Expand Up @@ -524,6 +525,7 @@ jobs:
neon-image:
needs: [ neon-image-arch, tag ]
if: "!(github.ref_name == 'release-compute' || github.base_ref == 'release-compute')"
runs-on: ubuntu-22.04
permissions:
id-token: write # aws-actions/configure-aws-credentials
Expand All @@ -545,6 +547,11 @@ jobs:
compute-node-image-arch:
needs: [ check-permissions, build-build-tools-image, tag ]
if: >-
!(
github.ref_name == 'release' || github.base_ref == 'release'
|| github.ref_name == 'release-proxy' || github.base_ref == 'release-proxy'
)
permissions:
id-token: write # aws-actions/configure-aws-credentials
statuses: write
Expand Down Expand Up @@ -636,6 +643,11 @@ jobs:
compute-node-image:
needs: [ compute-node-image-arch, tag ]
if: >-
!(
github.ref_name == 'release' || github.base_ref == 'release'
|| github.ref_name == 'release-proxy' || github.base_ref == 'release-proxy'
)
permissions:
id-token: write # aws-actions/configure-aws-credentials
statuses: write
Expand Down Expand Up @@ -678,6 +690,11 @@ jobs:
vm-compute-node-image:
needs: [ check-permissions, tag, compute-node-image ]
if: >-
!(
github.ref_name == 'release' || github.base_ref == 'release'
|| github.ref_name == 'release-proxy' || github.base_ref == 'release-proxy'
)
runs-on: [ self-hosted, large ]
strategy:
fail-fast: false
Expand Down Expand Up @@ -729,6 +746,8 @@ jobs:
test-images:
needs: [ check-permissions, tag, neon-image, compute-node-image ]
# Depends on jobs that can get skipped
if: "!failure() && !cancelled()"
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -773,7 +792,21 @@ jobs:
- name: Verify docker-compose example and test extensions
timeout-minutes: 20
env:
TAG: ${{needs.tag.outputs.build-tag}}
TAG: >-
${{
(github.ref_name == 'release-compute' || github.base_ref == 'release-compute')
&& needs.tag.outputs.previous-storage-release
|| needs.tag.outputs.build-tag
}}
COMPUTE_TAG: >-
${{
(
github.ref_name == 'release' || github.base_ref == 'release'
|| github.ref_name == 'release-proxy' || github.base_ref == 'release-proxy'
)
&& needs.tag.outputs.previous-compute-release
|| needs.tag.outputs.build-tag
}}
TEST_VERSION_ONLY: ${{ matrix.pg_version }}
run: ./docker-compose/docker_compose_test.sh

Expand Down Expand Up @@ -822,6 +855,7 @@ jobs:

push-neon-image-dev:
needs: [ generate-image-maps, neon-image ]
if: "!(github.ref_name == 'release-compute' || github.base_ref == 'release-compute')"
uses: ./.github/workflows/_push-to-container-registry.yml
with:
image-map: '${{ needs.generate-image-maps.outputs.neon-dev }}'
Expand All @@ -838,6 +872,11 @@ jobs:

push-compute-image-dev:
needs: [ generate-image-maps, vm-compute-node-image ]
if: >-
!(
github.ref_name == 'release' || github.base_ref == 'release'
|| github.ref_name == 'release-proxy' || github.base_ref == 'release-proxy'
)
uses: ./.github/workflows/_push-to-container-registry.yml
with:
image-map: '${{ needs.generate-image-maps.outputs.compute-dev }}'
Expand All @@ -853,7 +892,7 @@ jobs:
docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

push-neon-image-prod:
if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
if: github.ref_name == 'release' || github.ref_name == 'release-proxy'
needs: [ generate-image-maps, neon-image, test-images ]
uses: ./.github/workflows/_push-to-container-registry.yml
with:
Expand All @@ -870,7 +909,7 @@ jobs:
docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

push-compute-image-prod:
if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
if: github.ref_name == 'release-compute'
needs: [ generate-image-maps, vm-compute-node-image, test-images ]
uses: ./.github/workflows/_push-to-container-registry.yml
with:
Expand Down Expand Up @@ -982,7 +1021,7 @@ jobs:
deploy:
needs: [ check-permissions, push-neon-image-prod, push-compute-image-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait ]
# `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod`
# `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-neon-image-prod` and `push-compute-image-prod`
if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute') && !failure() && !cancelled()
permissions:
id-token: write # aws-actions/configure-aws-credentials
Expand Down Expand Up @@ -1161,7 +1200,7 @@ jobs:
id-token: write # aws-actions/configure-aws-credentials
statuses: write
contents: read
# `!failure() && !cancelled()` is required because the workflow transitively depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod`
# `!failure() && !cancelled()` is required because the workflow transitively depends on the job that can be skipped: `push-neon-image-prod` and `push-compute-image-prod`
if: github.ref_name == 'release' && !failure() && !cancelled()

runs-on: ubuntu-22.04
Expand Down Expand Up @@ -1251,7 +1290,8 @@ jobs:
pin-build-tools-image:
needs: [ build-build-tools-image, push-compute-image-prod, push-neon-image-prod, build-and-test-locally ]
if: github.ref_name == 'main'
# `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod`
if: github.ref_name == 'main' && !failure() && !cancelled()
uses: ./.github/workflows/pin-build-tools-image.yml
with:
from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }}
Expand Down
18 changes: 16 additions & 2 deletions .github/workflows/trigger-e2e-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,21 @@ jobs:
env:
EVENT_ACTION: ${{ github.event.action }}
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
TAG: ${{ needs.tag.outputs.build-tag }}
TAG: >-
${{
(github.ref_name == 'release-compute' || github.base_ref == 'release-compute')
&& needs.tag.outputs.previous-storage-release
|| needs.tag.outputs.build-tag
}}
COMPUTE_TAG: >-
${{
(
github.ref_name == 'release' || github.base_ref == 'release'
|| github.ref_name == 'release-proxy' || github.base_ref == 'release-proxy'
)
&& needs.tag.outputs.previous-compute-release
|| needs.tag.outputs.build-tag
}}
steps:
- name: Wait for `push-{neon,compute}-image-dev` job to finish
# It's important to have a timeout here, the script in the step can run infinitely
Expand Down Expand Up @@ -127,6 +141,6 @@ jobs:
--raw-field "commit_hash=$COMMIT_SHA" \
--raw-field "remote_repo=${GITHUB_REPOSITORY}" \
--raw-field "storage_image_tag=${TAG}" \
--raw-field "compute_image_tag=${TAG}" \
--raw-field "compute_image_tag=${COMPUTE_TAG}" \
--raw-field "concurrency_group=${E2E_CONCURRENCY_GROUP}" \
--raw-field "e2e-platforms=${E2E_PLATFORMS}"

0 comments on commit f95ede4

Please sign in to comment.