From cecb0c2f529fb447c0f1b81aa1cfe9ad458db863 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Fri, 28 Feb 2025 17:06:48 +0100 Subject: [PATCH] Multiplatform builds for docker by default. --- .github/workflows/docker-hub-platform.yml | 175 ---------------------- .github/workflows/docker-hub.yml | 170 ++++++++++++++++++--- .github/workflows/macos-test.yml | 4 +- 3 files changed, 148 insertions(+), 201 deletions(-) delete mode 100644 .github/workflows/docker-hub-platform.yml diff --git a/.github/workflows/docker-hub-platform.yml b/.github/workflows/docker-hub-platform.yml deleted file mode 100644 index 123c572b4..000000000 --- a/.github/workflows/docker-hub-platform.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Build and Push Docker Image for Multiple Architectures - -on: - pull_request: - push: - branches: - - master - - develop - tags: - - 'v*.*.*' - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - DOCKER_REGISTRY: index.docker.io - DOCKER_USERNAME: metacall - IMAGE_NAME: core - BUILDKIT_VERSION: 0.13.0 - -jobs: - build: - name: Build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/386 - - linux/arm64 - - linux/riscv64 - - linux/ppc64le - - linux/s390x - - linux/arm/v7 - - linux/arm/v6 - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker BuildX - uses: docker/setup-buildx-action@v3 - with: - version: v${{ env.BUILDKIT_VERSION }} - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Build MetaCall Docker Images - env: - METACALL_PLATFORM: ${{ matrix.platform }} - run: | - ./docker-compose.sh platform - - - name: Tag Platform Images - run: | - platform_tag=$(echo "${{ matrix.platform }}" | tr '/' '-') - echo "Platform Tag: ${platform_tag}" - for tag in "deps" "dev" "runtime" "cli"; do - docker tag metacall/${IMAGE_NAME}:${tag} \ - ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag} - done - - - name: Push Platform Images - run: | - platform_tag=$(echo "${{ matrix.platform }}" | tr '/' '-') - for tag in "deps" "dev" "runtime" "cli"; do - echo "Pushing image for tag: ${tag} with platform: ${platform_tag}" - docker push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag} - done - - - name: Run Tests - run: | - set -exuo pipefail - platform_tag=$(echo "${{ matrix.platform }}" | tr '/' '-') - cat < Dockerfile.test - FROM ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:cli-${platform_tag} - RUN echo "console.log('abcde')" > script.js - RUN metacallcli script.js - EOF - - docker build --platform ${{ matrix.platform }} -f Dockerfile.test -t test-image . - docker run --rm --platform=${{ matrix.platform }} test-image - - manifest: - name: Create and Push Manifest Lists - needs: build - # Only run when master or when tagging a version - if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.event_name != 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Create and Push Manifest Lists - run: | - tags=("deps" "dev" "runtime" "cli") - platforms=("linux/amd64" "linux/386" "linux/arm64" "linux/riscv64" "linux/ppc64le" "linux/s390x" "linux/arm/v7" "linux/arm/v6") - - echo "Create all the tags by platform" - - for tag in "${tags[@]}"; do - echo "Creating manifest for tag: $tag" - platform_tags="" - for platform in "${platforms[@]}"; do - platform_tag=$(echo "${platform}" | tr '/' '-') - platform_tags="${platform_tags} ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag}" - done - echo "Creating manifest with tags: ${platform_tags}" - docker manifest create ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag} ${platform_tags} --amend - docker manifest push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag} - done - - echo "Create the latest tag" - - cli_platform_tags="" - for platform in ${platforms[@]}"; do - platform_tag=$(echo "${platform}" | tr '/' '-') - cli_platform_tags="${cli_platform_tags} ${DOCKER_USERNAME}/${IMAGE_NAME}:cli-${platform_tag}" - done - docker manifest create ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:latest ${cli_platform_tags} --amend - docker manifest push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:latest - - if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" = true ]]; then - VERSION=${GITHUB_REF#refs/tags/v} - - echo "Create the version ${VERSION} tag" - - for tag in "${tags[@]}"; do - platform_tags="" - for platform in "${platforms[@]}"; do - platform_tag=$(echo "${platform}" | tr '/' '-') - platform_tags="${platform_tags} ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag}" - done - docker manifest create ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${VERSION}-${tag} ${platform_tags} --amend - docker manifest push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${VERSION}-${tag} - done - fi - - cleanup: - name: Cleanup Platform Specific Tags - needs: [build, manifest] - runs-on: ubuntu-latest - if: always() - steps: - - name: Remove Platform-Specific Tags - run: | - tags=("deps" "dev" "runtime" "cli") - platforms=("linux/amd64" "linux/386" "linux/arm64" "linux/riscv64" "linux/ppc64le" "linux/s390x" "linux/arm/v7" "linux/arm/v6") - - for tag in "${tags[@]}"; do - for platform in "${platforms[@]}"; do - platform_tag=$(echo "${platform}" | tr '/' '-') - tag_to_delete="${tag}-${platform_tag}" - - echo "Deleting tag: ${tag_to_delete}" - echo "https://hub.docker.com/v2/repositories/${DOCKER_USERNAME}/${IMAGE_NAME}/tags/${tag_to_delete}/" - - curl -X DELETE \ - -H "Authorization: Bearer ${{ secrets.DOCKER_HUB_ACCESS_TOKEN_DELETE }}" \ - "https://hub.docker.com/v2/repositories/${DOCKER_USERNAME}/${IMAGE_NAME}/tags/${tag_to_delete}/" - done - done diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 12375d985..123c572b4 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -1,13 +1,11 @@ -name: Build and Push Docker Image +name: Build and Push Docker Image for Multiple Architectures on: - # To enable manual triggering of this workflow - workflow_dispatch: - - # Trigger for pushes to master and tags + pull_request: push: branches: - master + - develop tags: - 'v*.*.*' @@ -16,38 +14,162 @@ concurrency: cancel-in-progress: true env: - IMAGE_NAME: index.docker.io/metacall/core + DOCKER_REGISTRY: index.docker.io + DOCKER_USERNAME: metacall + IMAGE_NAME: core + BUILDKIT_VERSION: 0.13.0 jobs: build: - runs-on: ubuntu-latest name: Build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - linux/amd64 + - linux/386 + - linux/arm64 + - linux/riscv64 + - linux/ppc64le + - linux/s390x + - linux/arm/v7 + - linux/arm/v6 steps: - - name: Checkout the code + - name: Checkout Repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Login to DockerHub - run: docker login -u "${{ secrets.DOCKER_HUB_USERNAME }}" -p "${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}" + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker BuildX + uses: docker/setup-buildx-action@v3 + with: + version: v${{ env.BUILDKIT_VERSION }} - - name: Pull MetaCall Docker Images - run: bash ./docker-compose.sh pull + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build MetaCall Docker Images - run: bash ./docker-compose.sh build + env: + METACALL_PLATFORM: ${{ matrix.platform }} + run: | + ./docker-compose.sh platform + + - name: Tag Platform Images + run: | + platform_tag=$(echo "${{ matrix.platform }}" | tr '/' '-') + echo "Platform Tag: ${platform_tag}" + for tag in "deps" "dev" "runtime" "cli"; do + docker tag metacall/${IMAGE_NAME}:${tag} \ + ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag} + done + + - name: Push Platform Images + run: | + platform_tag=$(echo "${{ matrix.platform }}" | tr '/' '-') + for tag in "deps" "dev" "runtime" "cli"; do + echo "Pushing image for tag: ${tag} with platform: ${platform_tag}" + docker push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag} + done - # TODO: Build with alpine and provide multiple tags (debian & alpine) once all tests pass - - name: Push MetaCall Docker Image to DockerHub + - name: Run Tests run: | - if [[ "${{ github.ref == 'refs/heads/master' }}" = true ]]; then - bash ./docker-compose.sh push - elif [[ "${{ contains(github.ref, 'refs/tags/') }}" = true ]]; then - bash ./docker-compose.sh version - else - echo "Failed to push the docker images" - exit 1 + set -exuo pipefail + platform_tag=$(echo "${{ matrix.platform }}" | tr '/' '-') + cat < Dockerfile.test + FROM ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:cli-${platform_tag} + RUN echo "console.log('abcde')" > script.js + RUN metacallcli script.js + EOF + + docker build --platform ${{ matrix.platform }} -f Dockerfile.test -t test-image . + docker run --rm --platform=${{ matrix.platform }} test-image + + manifest: + name: Create and Push Manifest Lists + needs: build + # Only run when master or when tagging a version + if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Create and Push Manifest Lists + run: | + tags=("deps" "dev" "runtime" "cli") + platforms=("linux/amd64" "linux/386" "linux/arm64" "linux/riscv64" "linux/ppc64le" "linux/s390x" "linux/arm/v7" "linux/arm/v6") + + echo "Create all the tags by platform" + + for tag in "${tags[@]}"; do + echo "Creating manifest for tag: $tag" + platform_tags="" + for platform in "${platforms[@]}"; do + platform_tag=$(echo "${platform}" | tr '/' '-') + platform_tags="${platform_tags} ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag}" + done + echo "Creating manifest with tags: ${platform_tags}" + docker manifest create ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag} ${platform_tags} --amend + docker manifest push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag} + done + + echo "Create the latest tag" + + cli_platform_tags="" + for platform in ${platforms[@]}"; do + platform_tag=$(echo "${platform}" | tr '/' '-') + cli_platform_tags="${cli_platform_tags} ${DOCKER_USERNAME}/${IMAGE_NAME}:cli-${platform_tag}" + done + docker manifest create ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:latest ${cli_platform_tags} --amend + docker manifest push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:latest + + if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" = true ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + + echo "Create the version ${VERSION} tag" + + for tag in "${tags[@]}"; do + platform_tags="" + for platform in "${platforms[@]}"; do + platform_tag=$(echo "${platform}" | tr '/' '-') + platform_tags="${platform_tags} ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${tag}-${platform_tag}" + done + docker manifest create ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${VERSION}-${tag} ${platform_tags} --amend + docker manifest push ${DOCKER_REGISTRY}/${DOCKER_USERNAME}/${IMAGE_NAME}:${VERSION}-${tag} + done fi - - name: Logout from DockerHub - run: docker logout + cleanup: + name: Cleanup Platform Specific Tags + needs: [build, manifest] + runs-on: ubuntu-latest + if: always() + steps: + - name: Remove Platform-Specific Tags + run: | + tags=("deps" "dev" "runtime" "cli") + platforms=("linux/amd64" "linux/386" "linux/arm64" "linux/riscv64" "linux/ppc64le" "linux/s390x" "linux/arm/v7" "linux/arm/v6") + + for tag in "${tags[@]}"; do + for platform in "${platforms[@]}"; do + platform_tag=$(echo "${platform}" | tr '/' '-') + tag_to_delete="${tag}-${platform_tag}" + + echo "Deleting tag: ${tag_to_delete}" + echo "https://hub.docker.com/v2/repositories/${DOCKER_USERNAME}/${IMAGE_NAME}/tags/${tag_to_delete}/" + + curl -X DELETE \ + -H "Authorization: Bearer ${{ secrets.DOCKER_HUB_ACCESS_TOKEN_DELETE }}" \ + "https://hub.docker.com/v2/repositories/${DOCKER_USERNAME}/${IMAGE_NAME}/tags/${tag_to_delete}/" + done + done diff --git a/.github/workflows/macos-test.yml b/.github/workflows/macos-test.yml index f46dd9256..64a768757 100644 --- a/.github/workflows/macos-test.yml +++ b/.github/workflows/macos-test.yml @@ -77,7 +77,7 @@ jobs: - name: Set up the environment run: sh ./tools/metacall-environment.sh $METACALL_INSTALL_OPTIONS env: - METACALL_INSTALL_OPTIONS: base python nodejs typescript c java ruby wasm rpc file cobol go backtrace #netcore5 c rust rapidjson funchook swig pack # clangformat v8rep51 coverage + METACALL_INSTALL_OPTIONS: base python nodejs typescript java ruby wasm rpc file cobol go backtrace #netcore5 c rust rapidjson funchook swig pack # clangformat v8rep51 coverage - name: Configure run: | @@ -85,7 +85,7 @@ jobs: . .env bash ../tools/metacall-configure.sh $METACALL_CONFIGURE_OPTIONS env: - METACALL_CONFIGURE_OPTIONS: ${{ matrix.options.build }} ${{ matrix.options.sanitizer }} scripts ports tests python nodejs typescript c java ruby wasm rpc file cobol go benchmarks install # netcore5 c rust examples pack # v8 coverage + METACALL_CONFIGURE_OPTIONS: ${{ matrix.options.build }} ${{ matrix.options.sanitizer }} scripts ports tests python nodejs typescript java ruby wasm rpc file cobol go benchmarks install # netcore5 c rust examples pack # v8 coverage - name: Build working-directory: ./build