From 1ba1b52e138ed08f487a66f8ebb40186e7ef44cb Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 16 Jul 2024 08:37:36 +0000 Subject: [PATCH 1/2] Don't pull and save all the images every build We only need to pull and save images when building tags, if building an untagged dev release, just inspect to make sure the images all exist Signed-off-by: Brad Davidson --- Dockerfile | 1 + scripts/build-images | 31 ++++++++++++++++++++----------- scripts/build-windows-images | 9 +++++---- scripts/package-images | 9 ++++++++- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index c27c073d65..76c68c78f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ RUN set -x && \ rsync \ gcc \ bsd-compat-headers \ + skopeo \ py-pip \ py3-pip \ pigz \ diff --git a/scripts/build-images b/scripts/build-images index 267753f096..15866ad07a 100755 --- a/scripts/build-images +++ b/scripts/build-images @@ -7,11 +7,20 @@ source ./scripts/version.sh ./scripts/build-image-runtime -awk '{print $1}' << EOF > build/images-core.txt - ${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION} -EOF +echo ${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION} > build/images-core.txt + +# If not building a release, inspect the images to ensure they exist instead of +# doing a full pull, and only include the runtime image in the core image list. +# The core image list is saved to a tarball and used later in tests. +if [[ $RKE2_PATCH == dev.* ]]; then + IMAGE_COMMAND='skopeo inspect --raw docker://$0 >/dev/null && echo $0' + IMAGES_CORE=/dev/null +else + IMAGE_COMMAND='docker image pull --quiet $0' + IMAGES_CORE=build/images-core.txt +fi -xargs -n1 -t docker image pull --quiet << EOF >> build/images-core.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF >> $IMAGES_CORE ${REGISTRY}/rancher/hardened-kubernetes:${KUBERNETES_IMAGE_TAG} ${REGISTRY}/rancher/hardened-coredns:v1.11.1-build20240305 ${REGISTRY}/rancher/hardened-cluster-autoscaler:v1.8.10-build20240124 @@ -29,13 +38,13 @@ xargs -n1 -t docker image pull --quiet << EOF >> build/images-core.txt ${REGISTRY}/rancher/mirrored-sig-storage-snapshot-validation-webhook:v6.2.2 EOF -xargs -n1 -t docker image pull --quiet << EOF > build/images-canal.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-canal.txt ${REGISTRY}/rancher/hardened-calico:v3.28.0-build20240625 ${REGISTRY}/rancher/hardened-flannel:v0.25.4-build20240610 EOF if [ "${GOARCH}" != "s390x" ]; then -xargs -n1 -t docker image pull --quiet << EOF > build/images-cilium.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-cilium.txt ${REGISTRY}/rancher/mirrored-cilium-certgen:v0.1.12 ${REGISTRY}/rancher/mirrored-cilium-cilium:v1.15.5 ${REGISTRY}/rancher/mirrored-cilium-cilium-envoy:v1.28.3-31ec52ec5f2e4d28a8e19a0bfb872fa48cf7a515 @@ -49,7 +58,7 @@ xargs -n1 -t docker image pull --quiet << EOF > build/images-cilium.txt ${REGISTRY}/rancher/hardened-cni-plugins:v1.4.1-build20240325 EOF -xargs -n1 -t docker image pull --quiet << EOF > build/images-calico.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-calico.txt ${REGISTRY}/rancher/mirrored-calico-operator:v1.32.7 ${REGISTRY}/rancher/mirrored-calico-ctl:v3.27.3 ${REGISTRY}/rancher/mirrored-calico-kube-controllers:v3.27.3 @@ -63,7 +72,7 @@ xargs -n1 -t docker image pull --quiet << EOF > build/images-calico.txt EOF if [ "${GOARCH}" != "arm64" ]; then -xargs -n1 -t docker image pull --quiet << EOF > build/images-vsphere.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-vsphere.txt ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-cpi-release-manager:v1.30.1 ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-csi-release-driver:v3.3.0 ${REGISTRY}/rancher/mirrored-cloud-provider-vsphere-csi-release-syncer:v3.3.0 @@ -76,7 +85,7 @@ xargs -n1 -t docker image pull --quiet << EOF > build/images-vsphere.txt EOF fi -xargs -n1 -t docker image pull --quiet << EOF > build/images-multus.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-multus.txt ${REGISTRY}/rancher/hardened-multus-cni:v4.0.2-build20240612 ${REGISTRY}/rancher/hardened-cni-plugins:v1.4.1-build20240430 ${REGISTRY}/rancher/hardened-node-feature-discovery:v0.15.4-build20240513 @@ -91,7 +100,7 @@ xargs -n1 -t docker image pull --quiet << EOF > build/images-multus.txt ${REGISTRY}/rancher/mirrored-library-busybox:1.36.1 EOF -xargs -n1 -t docker image pull --quiet << EOF > build/images-harvester.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-harvester.txt ${REGISTRY}/rancher/harvester-cloud-provider:v0.2.1 ${REGISTRY}/rancher/mirrored-kube-vip-kube-vip-iptables:v0.6.0 ${REGISTRY}/rancher/harvester-csi-driver:v0.1.6 @@ -101,7 +110,7 @@ xargs -n1 -t docker image pull --quiet << EOF > build/images-harvester.txt ${REGISTRY}/rancher/mirrored-longhornio-csi-attacher:v3.2.1 EOF -xargs -n1 -t docker image pull --quiet << EOF > build/images-flannel.txt +xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-flannel.txt ${REGISTRY}/rancher/hardened-flannel:v0.25.4-build20240610 ${REGISTRY}/rancher/hardened-cni-plugins:v1.4.1-build20240430 EOF diff --git a/scripts/build-windows-images b/scripts/build-windows-images index 12a3bcf8eb..fc37811560 100755 --- a/scripts/build-windows-images +++ b/scripts/build-windows-images @@ -11,7 +11,8 @@ fi mkdir -p build -WINDOWS_IMAGES=(${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-windows-amd64 rancher/mirrored-pause:${PAUSE_VERSION}-windows-1809-amd64 rancher/mirrored-pause:${PAUSE_VERSION}-windows-ltsc2022-amd64) -for IMAGE in "${WINDOWS_IMAGES[@]}"; do - echo "${IMAGE}" >> build/windows-images.txt -done +cat << EOF > build/windows-images.txt +${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-windows-amd64 +${REGISTRY}/rancher/mirrored-pause:${PAUSE_VERSION}-windows-1809-amd64 +${REGISTRY}/rancher/mirrored-pause:${PAUSE_VERSION}-windows-ltsc2022-amd64 +EOF diff --git a/scripts/package-images b/scripts/package-images index 0ac0c8eff6..cde53c1c1d 100755 --- a/scripts/package-images +++ b/scripts/package-images @@ -7,10 +7,17 @@ source ./scripts/version.sh mkdir -p dist/artifacts +# If not building a release, only save the core image tarball +if [[ $RKE2_PATCH == dev.* ]]; then + IMAGE_LISTS=(build/images-core.txt) +else + IMAGE_LISTS=(build/images*.txt) +fi + # We reorder the tar file so that the metadata files are at the start of the archive, which should make loading # the runtime image faster. By default `docker image save` puts these at the end of the archive, which means the entire # tarball needs to be read even if you're just loading a single image. -for FILE in build/images*.txt; do +for FILE in "${IMAGE_LISTS[@]}" do BASE=$(basename ${FILE} .txt) DEST=build/images/${PROG}-${BASE}.tar docker image save --output ${DEST}.tmp $(<${FILE}) From b9902685380a135aaac627c1438ffa58379864e5 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 16 Jul 2024 17:49:22 +0000 Subject: [PATCH 2/2] Use mirror.gcr.io as buildx pull-through cache Signed-off-by: Brad Davidson --- .github/workflows/build.yml | 51 +++++++++++-- .github/workflows/pr.yml | 45 +++++++++-- .github/workflows/release.yml | 123 +++++++++++++++++++++++++----- .github/workflows/test-suite.yaml | 19 ++++- Dockerfile | 79 ++++++------------- Dockerfile.windows | 14 ++-- Makefile | 6 +- scripts/build-image-runtime | 23 ++++-- scripts/build-image-test | 10 ++- scripts/build-images | 9 ++- scripts/dev-shell-build | 2 +- 11 files changed, 270 insertions(+), 111 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e0bbc5eb3..8ee359a471 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,9 +19,27 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + - name: Export Cache Env Vars + uses: actions/github-script@v7 + env: + github-token: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper - name: "Read secrets" uses: rancher-eio/read-vault-secrets@main @@ -31,21 +49,42 @@ jobs: secret/data/github/repo/${{ github.repository }}/aws-secret-access-key/credentials token | AWS_SECRET_ACCESS_KEY ; - name: Build run: | - dapper -f Dockerfile --target dapper make dapper-ci + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dapper-ci env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Test run: | - dapper -f Dockerfile --target dapper make test + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make test + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} build-arm64: runs-on: runs-on,runner=8cpu-linux-arm64,run-id=${{ github.run_id }},image=ubuntu22-full-arm64,hdd=64 steps: - name: Checkout code uses: actions/checkout@v4 + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + - name: Export Cache Env Vars + uses: actions/github-script@v7 + env: + github-token: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper - name: "Read secrets" uses: rancher-eio/read-vault-secrets@main @@ -55,8 +94,8 @@ jobs: secret/data/github/repo/${{ github.repository }}/aws-secret-access-key/credentials token | AWS_SECRET_ACCESS_KEY ; - name: Build run: | - dapper -f Dockerfile --target dapper make dapper-ci + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dapper-ci env: AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - \ No newline at end of file + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3eb84d1e96..db900395b7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -15,26 +15,59 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + - name: Export Cache Env Vars + uses: actions/github-script@v7 + env: + github-token: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper - name: Build run: | - dapper -f Dockerfile --target dapper make dapper-ci + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dapper-ci + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} + SKIP_DEV_RPM: true - name: Test run: | - dapper -f Dockerfile --target dapper make test + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make test + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} build-arm64: runs-on: runs-on,runner=8cpu-linux-arm64,run-id=${{ github.run_id }},image=ubuntu22-full-arm64,hdd=64 steps: - name: Checkout code uses: actions/checkout@v4 + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper - name: Build run: | - dapper -f Dockerfile --target dapper make dapper-ci - \ No newline at end of file + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dapper-ci + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index abbf3ee746..cf32fbc2b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,16 +25,40 @@ jobs: - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + + - name: Export Cache Env Vars + uses: actions/github-script@v7 + env: + github-token: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) + - name: Validate Release run: | - dapper -f Dockerfile --target dapper make validate-release + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make validate-release + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Build run: | - dapper -f Dockerfile --target dapper make dapper-ci + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dapper-ci + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: "Read secrets" uses: rancher-eio/read-vault-secrets@main @@ -45,27 +69,38 @@ jobs: - name: Package Images run: | - dapper -f Dockerfile --target dapper make package-images + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make package-images + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Scan Images continue-on-error: true run: | - dapper -f Dockerfile --target dapper make scan-images + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make scan-images + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Test run: | - dapper -f Dockerfile --target dapper make test + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make test + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Publish Image Runtime run: | - GITHUB_ACTION_TAG=${{ github.ref_name }} dapper -f Dockerfile --target dapper make publish-image-runtime + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make publish-image-runtime env: DOCKER_USERNAME: ${{ env.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ env.DOCKER_PASSWORD }} + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} + GITHUB_ACTION_TAG: ${{ github.ref_name }} - name: Checksum Artifacts run: | - GITHUB_ACTION_TAG=${{ github.ref_name }} dapper -f Dockerfile --target dapper make checksum + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make checksum + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} + GITHUB_ACTION_TAG: ${{ github.ref_name }} - name: Publish Artifacts run: | @@ -76,18 +111,42 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + + - name: Export Cache Env Vars + uses: actions/github-script@v7 + env: + github-token: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) + - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper - name: Validate Release run: | - dapper -f Dockerfile --target dapper make validate-release + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make validate-release + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Build run: | - dapper -f Dockerfile --target dapper make dapper-ci + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dapper-ci + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: "Read secrets" uses: rancher-eio/read-vault-secrets@main @@ -98,23 +157,32 @@ jobs: - name: Package Images run: | - dapper -f Dockerfile --target dapper make package-images + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make package-images + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Scan Images continue-on-error: true run: | - dapper -f Dockerfile --target dapper make scan-images + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make scan-images + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} - name: Publish Image Runtime run: | - GITHUB_ACTION_TAG=${{ github.ref_name }} dapper -f Dockerfile --target dapper make publish-image-runtime + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make publish-image-runtime env: DOCKER_USERNAME: ${{ env.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ env.DOCKER_PASSWORD }} + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} + GITHUB_ACTION_TAG: ${{ github.ref_name }} - name: Checksum run: | - GITHUB_ACTION_TAG=${{ github.ref_name }} dapper -f Dockerfile --target dapper make checksum + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make checksum + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} + GITHUB_ACTION_TAG: ${{ github.ref_name }} - name: Publish Artifacts run: | @@ -126,9 +194,29 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] + + - name: Export Cache Env Vars + uses: actions/github-script@v7 + env: + github-token: ${{ secrets.GITHUB_TOKEN }} + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL']) + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN']) + core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL']) + - name: Install Dapper run: | - curl -sL https://releases.rancher.com/dapper/latest/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper + curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$(uname -s)-$(uname -m) > /usr/local/bin/dapper chmod +x /usr/local/bin/dapper - name: "Read secrets" @@ -139,8 +227,9 @@ jobs: - name: Dispatch run: | - dapper -f Dockerfile --target dapper make dispatch + dapper -f Dockerfile --bake --cache-from type=gha --cache-to type=gha,ignore-error=true,mode=max --target dapper make dispatch env: PAT_TOKEN: ${{ secrets.GITHUB_TOKEN }} PATH_USERNAME: ${{ env.PAT_USERNAME }} GITHUB_ACTION_TAG: ${{ env.GITHUB_ACTION_TAG }} + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 58d3588f15..461e55d411 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -28,6 +28,15 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: buildx + with: + version: v0.16.0 + driver: docker-container + buildkitd-config-inline: | + [registry."docker.io"] + mirrors = ["mirror.gcr.io"] - name: Find Go Version for Build id: go-finder run: | @@ -39,8 +48,6 @@ jobs: uses: ./.github/actions/setup-go with: go-version: ${{ steps.go-finder.outputs.VERSION_GOLANG }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - name: Install OS Packages run: sudo apt-get install -y libarchive-tools g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 # Can only upload from a single path, so we need to copy the binary to the image directory @@ -48,11 +55,15 @@ jobs: # just compressed. We remove the rke2-runtime.tar as its not used by the install script. - name: Build RKE2 Binary and Compressed Runtime Image run: | - GOCOVER=true make package-bundle + make package-bundle make package-image-runtime cp ./bin/rke2 ./build/images/rke2 cp ./dist/artifacts/rke2.*-amd64.tar.gz ./build/images/ rm ./build/images/rke2-runtime.tar + env: + BUILDX_BUILDER: ${{ steps.buildx.outputs.name }} + GOCOVER: "true" + - name: Upload RKE2 Binary and Runtime Image uses: actions/upload-artifact@v4 with: @@ -175,4 +186,4 @@ jobs: with: ## If no one connects after 5 minutes, shut down server. wait-timeout-minutes: 5 - limit-access-to-actor: true \ No newline at end of file + limit-access-to-actor: true diff --git a/Dockerfile b/Dockerfile index 76c68c78f7..9dfc35bd0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG KUBERNETES_VERSION=dev # Build environment FROM rancher/hardened-build-base:v1.22.4b1 AS build ARG DAPPER_HOST_ARCH -ENV ARCH $DAPPER_HOST_ARCH +ENV ARCH="$DAPPER_HOST_ARCH" RUN set -x && \ apk --no-cache add \ bash \ @@ -31,48 +31,26 @@ RUN zypper install -y systemd-rpm-macros # Dapper/Drone/CI environment FROM build AS dapper -ENV DAPPER_ENV GODEBUG GOCOVER REPO TAG GITHUB_ACTION_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY DOCKER_USERNAME DOCKER_PASSWORD ARG DAPPER_HOST_ARCH -ENV ARCH $DAPPER_HOST_ARCH -ENV DAPPER_OUTPUT ./dist ./bin ./build -ENV DAPPER_DOCKER_SOCKET true -ENV DAPPER_TARGET dapper -ENV DAPPER_RUN_ARGS "--privileged --network host -v /tmp:/tmp -v rke2-pkg:/go/pkg -v rke2-cache:/root/.cache/go-build -v trivy-cache:/root/.cache/trivy" -RUN if [ "${ARCH}" = "amd64" ] || [ "${ARCH}" = "arm64" ]; then \ - VERSION=0.56.10 OS=linux && \ - curl -sL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_${ARCH}.tar.gz" | \ - tar -xzf - -C /usr/local/bin; \ - fi -RUN curl -sL https://dl.k8s.io/release/$( \ - curl -sL https://dl.k8s.io/release/stable.txt \ - )/bin/linux/${ARCH}/kubectl -o /usr/local/bin/kubectl && \ - chmod a+x /usr/local/bin/kubectl; \ - pip install codespell +ENV ARCH="$DAPPER_HOST_ARCH" +ENV DAPPER_ENV="GODEBUG GOCOVER REPO TAG SKIP_DEV_RPM GITHUB_ACTION_TAG ACTIONS_CACHE_URL ACTIONS_RUNTIME_TOKEN PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION BUILDX_BUILDER DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY DOCKER_USERNAME DOCKER_PASSWORD" +ENV DAPPER_OUTPUT="./dist ./bin ./build" +ENV DAPPER_DOCKER_SOCKET="true" +ENV DAPPER_TARGET="dapper" +ENV DAPPER_RUN_ARGS="--privileged --network host -v /home/runner/.docker:/root/.docker -v /tmp:/tmp -v rke2-pkg:/go/pkg -v rke2-cache:/root/.cache/go-build -v trivy-cache:/root/.cache/trivy" +RUN curl -fsL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.55.2 +RUN VERSION=0.56.10 OS=linux && \ + curl -fsL "https://github.com/vmware-tanzu/sonobuoy/releases/download/v${VERSION}/sonobuoy_${VERSION}_${OS}_${ARCH}.tar.gz" | \ + tar -xzvC /usr/local/bin +RUN VERSION=$(curl -fsL https://dl.k8s.io/release/stable.txt) && \ + curl -fsL "https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubectl" -o /usr/local/bin/kubectl && \ + chmod a+x /usr/local/bin/kubectl +RUN VERSION=v0.20.1 ARCH=$(bash -c 'echo ${ARCH/amd64/x86_64}') && \ + curl -fsL "https://github.com/google/go-containerregistry/releases/download/${VERSION}/go-containerregistry_Linux_${ARCH}.tar.gz" | \ + tar -zxvC /usr/local/bin crane -RUN python3 -m pip install awscli -RUN curl -sL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.55.2 -RUN set -x && \ - apk --no-cache add \ - libarchive-tools \ - zstd \ - jq \ - python3 && \ - if [ "${ARCH}" != "s390x" ] || [ "${GOARCH}" != "arm64" ]; then \ - apk add --no-cache rpm-dev; \ - fi - -RUN GOCR_VERSION="v0.5.1" && \ - if [ "${ARCH}" = "arm64" ]; then \ - wget https://github.com/google/go-containerregistry/releases/download/${GOCR_VERSION}/go-containerregistry_Linux_arm64.tar.gz && \ - tar -zxvf go-containerregistry_Linux_arm64.tar.gz && \ - mv crane /usr/local/bin && \ - chmod a+x /usr/local/bin/crane; \ - else \ - wget https://github.com/google/go-containerregistry/releases/download/${GOCR_VERSION}/go-containerregistry_Linux_x86_64.tar.gz && \ - tar -zxvf go-containerregistry_Linux_x86_64.tar.gz && \ - mv crane /usr/local/bin && \ - chmod a+x /usr/local/bin/crane; \ - fi +RUN apk --no-cache add libarchive-tools zstd jq rpm-dev python3 +RUN python3 -m pip install awscli codespell WORKDIR /source @@ -81,22 +59,13 @@ COPY --from=rpm-macros /usr/lib/rpm/macros.d/macros.systemd /usr/lib/rpm/macros. # Shell used for debugging FROM dapper AS shell -RUN set -x && \ - apk --no-cache add \ - bash-completion \ - iptables \ - less \ - psmisc \ - rsync \ - socat \ - sudo \ - vim +RUN apk --no-cache add bash-completion iptables less psmisc rsync socat sudo vim # For integration tests RUN go get github.com/onsi/ginkgo/v2 github.com/onsi/gomega/... RUN GO111MODULE=off GOBIN=/usr/local/bin go get github.com/go-delve/delve/cmd/dlv RUN echo 'alias abort="echo -e '\''q\ny\n'\'' | dlv connect :2345"' >> /root/.bashrc -ENV PATH=/var/lib/rancher/rke2/bin:$PATH -ENV KUBECONFIG=/etc/rancher/rke2/rke2.yaml +ENV PATH="/var/lib/rancher/rke2/bin:$PATH" +ENV KUBECONFIG="/etc/rancher/rke2/rke2.yaml" VOLUME /var/lib/rancher/rke2 # This makes it so we can run and debug k3s too VOLUME /var/lib/rancher/k3s @@ -156,9 +125,9 @@ COPY build/images/rke2-images.linux-amd64.tar.zst /var/lib/rancher/rke2/agent/im COPY build/images.txt /images.txt # use rke2 bundled binaries -ENV PATH=/var/lib/rancher/rke2/bin:$PATH +ENV PATH="/var/lib/rancher/rke2/bin:$PATH" # for kubectl -ENV KUBECONFIG=/etc/rancher/rke2/rke2.yaml +ENV KUBECONFIG="/etc/rancher/rke2/rke2.yaml" # for crictl ENV CONTAINER_RUNTIME_ENDPOINT="unix:///run/k3s/containerd/containerd.sock" # for ctr diff --git a/Dockerfile.windows b/Dockerfile.windows index 822e2363a2..fea2c07898 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -6,13 +6,13 @@ RUN apk --no-cache add \ # Dapper/Drone/CI environment FROM rancher/hardened-build-base:v1.21.5b2 AS dapper -ENV DAPPER_ENV GODEBUG REPO TAG GITHUB_ACTION_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY ARG DAPPER_HOST_ARCH -ENV ARCH $DAPPER_HOST_ARCH -ENV DAPPER_OUTPUT ./dist ./bin ./build -ENV DAPPER_DOCKER_SOCKET true -ENV DAPPER_TARGET dapper -ENV DAPPER_RUN_ARGS "--privileged --network host -v /tmp:/tmp -v rke2-pkg:/go/pkg -v rke2-cache:/root/.cache/go-build" +ENV ARCH="$DAPPER_HOST_ARCH" +ENV DAPPER_ENV="GODEBUG REPO TAG GITHUB_ACTION_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DRONE_BUILD_EVENT IMAGE_NAME AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID ENABLE_REGISTRY" +ENV DAPPER_OUTPUT="./dist ./bin ./build" +ENV DAPPER_DOCKER_SOCKET="true" +ENV DAPPER_TARGET="dapper" +ENV DAPPER_RUN_ARGS="--privileged --network host -v /home/runner/.docker:/root/.docker -v /tmp:/tmp -v rke2-pkg:/go/pkg -v rke2-cache:/root/.cache/go-build" RUN apk update RUN set -x && \ apk add --no-cache \ @@ -39,7 +39,7 @@ WORKDIR /source # End Dapper stuff FROM rancher/hardened-containerd:v1.7.17-k3s1-build20240605-amd64-windows AS containerd -FROM build as windows-runtime-collect +FROM build AS windows-runtime-collect ARG KUBERNETES_VERSION=dev # windows runtime image diff --git a/Makefile b/Makefile index 699d52cd24..247e576e22 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ ci-shell: clean .dapper ## Launch a shell in the CI environment @echo '# Run "make dapper-ci" to reproduce CI in this shell #' @echo '######################################################' @echo - ./.dapper -f Dockerfile --target dapper -s + ./.dapper --bake -f Dockerfile --target dapper -s .PHONY: dapper-ci dapper-ci: .ci ## Used by Drone CI, does the same as "ci" but in a Drone way @@ -157,14 +157,14 @@ checksum: ./.dapper: @echo Downloading dapper - @curl -sL https://releases.rancher.com/dapper/v0.5.8/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp + @curl -sL https://github.com/brandond/dapper/releases/download/v0.7.0-bd5/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp @@chmod +x .dapper.tmp @./.dapper.tmp -v @mv .dapper.tmp .dapper in-docker-%: .dapper ## Advanced: wraps any target in Docker environment, for example: in-docker-build-debug mkdir -p ./bin/ ./dist/ ./build - ./.dapper -f Dockerfile --target dapper make $* + ./.dapper --bake -f Dockerfile --target dapper make $* .PHONY: help help: ## this help diff --git a/scripts/build-image-runtime b/scripts/build-image-runtime index c11896735e..239850526a 100755 --- a/scripts/build-image-runtime +++ b/scripts/build-image-runtime @@ -5,7 +5,10 @@ cd $(dirname $0)/.. source ./scripts/version.sh -DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} docker image build \ +docker buildx build \ + --load \ + --cache-from type=gha \ + --cache-to type=gha,ignore-error=true,mode=max \ --build-arg TAG=${VERSION} \ --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} \ --build-arg MAJOR=${VERSION_MAJOR} \ @@ -18,8 +21,16 @@ DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} docker image build \ --file Dockerfile \ . +mkdir -p build/images +docker image save \ + --output build/images/${PROG}-runtime.tar \ + ${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-${GOOS}-${GOARCH} + if [ "${GOARCH}" != "s390x" ] && [ "${GOARCH}" != "arm64" ]; then - DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} docker image build \ + docker buildx build \ + --load \ + --cache-from type=gha \ + --cache-to type=gha,ignore-error=true,mode=max \ --build-arg TAG=${VERSION} \ --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} \ --build-arg MAJOR=${VERSION_MAJOR} \ @@ -30,12 +41,10 @@ if [ "${GOARCH}" != "s390x" ] && [ "${GOARCH}" != "arm64" ]; then --file Dockerfile.windows \ . # Only ever used in its compressed form for e2e tests - mkdir -p build/images docker image save \ ${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-windows-${GOARCH} | \ zstd -T0 -16 -f --long=25 --no-progress - -o build/images/${PROG}-images.windows-${GOARCH}.tar.zst fi -mkdir -p build/images -docker image save \ - --output build/images/${PROG}-runtime.tar \ - ${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION}-${GOOS}-${GOARCH} + +# fix builder activity timestamp file ownership getting broken when mounted into the dapper container +rm ~/.docker/buildx/activity/* diff --git a/scripts/build-image-test b/scripts/build-image-test index a3b7ca670a..f1a72ef5b5 100755 --- a/scripts/build-image-test +++ b/scripts/build-image-test @@ -9,11 +9,17 @@ if [ "${GOARCH}" == "s390x" ] || [ "${GOARCH}" == "arm64" ]; then exit 0 fi -DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} docker image build \ +docker buildx build \ + --load \ + --cache-from type=gha \ + --cache-to type=gha,ignore-error=true,mode=max \ --build-arg TAG=${VERSION} \ --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} \ --build-arg CACHEBUST="$(date +%s%N)" \ --tag ${REPO}/${PROG}-test:${DOCKERIZED_VERSION} \ --tag ${REPO}/${PROG}-test:${DOCKERIZED_VERSION}-${GOARCH} \ --target test \ -. + . + +# fix builder activity timestamp file ownership getting broken when mounted into the dapper container +rm ~/.docker/buildx/activity/* diff --git a/scripts/build-images b/scripts/build-images index 15866ad07a..fed036fc24 100755 --- a/scripts/build-images +++ b/scripts/build-images @@ -14,7 +14,7 @@ echo ${REGISTRY}/${REPO}/${PROG}-runtime:${DOCKERIZED_VERSION} > build/images-co # The core image list is saved to a tarball and used later in tests. if [[ $RKE2_PATCH == dev.* ]]; then IMAGE_COMMAND='skopeo inspect --raw docker://$0 >/dev/null && echo $0' - IMAGES_CORE=/dev/null + IMAGES_CORE=/tmp/images-core.txt else IMAGE_COMMAND='docker image pull --quiet $0' IMAGES_CORE=build/images-core.txt @@ -115,5 +115,8 @@ xargs -n1 sh -xc "$IMAGE_COMMAND" << EOF > build/images-flannel.txt ${REGISTRY}/rancher/hardened-cni-plugins:v1.4.1-build20240430 EOF fi -# Continue to provide a legacy airgap archive set with the default CNI images -cat build/images-core.txt build/images-canal.txt > build/images.txt + +# Continue to provide a legacy airgap archive set with the default CNI images. +# This file is also used by the airgap image test to confirm that we're capturing +# the correct files for a cluster with default features. +cat /tmp/images-core.txt build/images-core.txt build/images-canal.txt > build/images.txt || true diff --git a/scripts/dev-shell-build b/scripts/dev-shell-build index 48c8d1e0e2..999ec2b646 100755 --- a/scripts/dev-shell-build +++ b/scripts/dev-shell-build @@ -10,4 +10,4 @@ if [ ! -d build/images ]; then fi # build the dev shell image -DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} docker image build -t ${PROG}-dev --target shell . +docker buildx build --load -t ${PROG}-dev --target shell .