Push images to Public ECR

.buildkite/pipeline.yaml

@@ -1,6 +1,6 @@
 x-anchors:
-  push: &push
-    label: ":helm::docker: push controller image and helm chart"
+  push-helm: &push-helm
+    label: ":helm::docker: build and push helm chart"
     plugins:
       - kubernetes:
           checkout:
@@ -10,7 +10,7 @@ x-anchors:
             containers:
               - name: deploy
                 image: alpine:latest
-                command: [.buildkite/steps/build-and-push.sh]
+                command: [.buildkite/steps/build-and-push-helm.sh]
                 envFrom:
                   - secretRef:
                       name: deploy-secrets
@@ -162,15 +162,15 @@ steps:
           files: junit-*.xml
           format: junit
 
-  - label: ":docker: build controller"
+  - label: ":docker: build and push controller image"
     key: controller
     plugins:
       - kubernetes:
           podSpec:
             containers:
               - name: ko
                 image: golang:1.23
-                command: [.buildkite/steps/controller.sh]
+                command: [.buildkite/steps/build-and-push-controller.sh]
                 envFrom:
                   - secretRef:
                       name: deploy-secrets
@@ -181,7 +181,7 @@ steps:
   # On feature branches, don't wait for tests. We may want to deploy
   # to a test cluster to debug the feature branch.
   - if: build.branch != pipeline.default_branch && build.tag !~ /^.+\$/
-    <<: *push
+    <<: *push-helm
     key: push-feature-branch
     depends_on:
       - tidy
@@ -193,7 +193,7 @@ steps:
   # On the main branch or tags, wait for tests. We don't want to
   # push a new image or chart unless the tests pass.
   - if: build.branch == pipeline.default_branch || build.tag =~ /^.+\$/
-    <<: *push
+    <<: *push-helm
     key: push-main-or-tag
     depends_on:
       - tidy

.buildkite/steps/assume-role.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+set -eu
+
+# TODO: design config to apply a plugin only to the checkout container, command
+# containers, or a specific container(s)...
+# TODO: then replace this script with aws-assume-role-with-web-identity plugin
+
+echo "~~~ :buildkite::key::aws: Requesting an OIDC token for AWS from Buildkite"
+
+role_arn='arn:aws:iam::172840064832:role/pipeline-buildkite-kubernetes-stack-kubernetes-agent-stack'
+BUILDKITE_OIDC_TOKEN="$(buildkite-agent oidc request-token --audience sts.amazonaws.com)"
+
+echo "~~~ :aws: Assuming role using OIDC token"
+ASSUME_ROLE_RESPONSE="$(aws sts assume-role-with-web-identity \
+    --role-arn "${role_arn}" \
+    --role-session-name "buildkite-job-${BUILDKITE_JOB_ID}" \
+    --web-identity-token "${BUILDKITE_OIDC_TOKEN}"
+)"
+ASSUME_ROLE_CMD_STATUS=$?
+
+if [ "${ASSUME_ROLE_CMD_STATUS}" -ne 0 ]; then
+    echo "^^^ +++"
+    echo "Failed to assume AWS role:"
+    echo "${ASSUME_ROLE_RESPONSE}"
+    exit 1
+fi
+
+AWS_ACCESS_KEY_ID="$(echo "${ASSUME_ROLE_RESPONSE}" | jq -r ".Credentials.AccessKeyId")"
+AWS_SECRET_ACCESS_KEY="$(echo "${ASSUME_ROLE_RESPONSE}" | jq -r ".Credentials.SecretAccessKey")"
+AWS_SESSION_TOKEN="$(echo "${ASSUME_ROLE_RESPONSE}" | jq -r ".Credentials.SessionToken")"
+export AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY
+export AWS_SESSION_TOKEN

.buildkite/steps/build-and-push-controller.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -Eeufo pipefail
+
+echo --- Installing packages
+apt-get update && apt-get install -y --no-install-recommends awscli jq
+
+echo --- Installing ko
+KO_VERSION="0.13.0"
+OS="$(go env GOOS)"
+ARCH="$(uname -m)"
+curl -sSfL "https://github.com/ko-build/ko/releases/download/v${KO_VERSION}/ko_${KO_VERSION}_${OS^}_${ARCH}.tar.gz" | tar -xzv -C /bin ko
+
+tag="$(git describe)"
+version="${tag#v}"
+
+# Is this a release version (version-tagged)?
+if [[ "${version}" == "${BUILDKITE_TAG#v}" ]] ; then
+    # Publish to both PECR and GHCR
+    controller_repo_pecr="public.ecr.aws/buildkite/agent-stack-k8s/controller"
+    controller_repo_ghcr="ghcr.io/buildkite/agent-stack-k8s/controller"
+else
+    # Publish dev images to PECR dev repo only
+    controller_repo_pecr="public.ecr.aws/buildkite/agent-stack-k8s-dev/controller"
+fi
+
+if [[ "${controller_repo_pecr:-}" != "" ]] ; then
+    . .buildkite/steps/assume-role.sh
+
+    echo ~~~ Logging into Public ECR
+    ko login public.ecr.aws -u AWS --password "$(aws --region us-east-1 ecr-public get-login-password)"
+
+    echo --- Building with ko for Public ECR
+    controller_image_pecr="$(
+        VERSION="${tag}" \
+        KO_DOCKER_REPO="${controller_repo_pecr}" \
+            ko build --bare --tags "${version}" --platform linux/amd64,linux/arm64 \
+    )"
+    buildkite-agent meta-data set controller-image-pecr "${controller_image_pecr}"
+fi
+if [[ "${controller_repo_ghcr:-}" != "" ]] ; then
+    echo --- Logging into to GHCR
+    ko login ghcr.io -u "${REGISTRY_USERNAME}" --password "${REGISTRY_PASSWORD}"
+
+    echo --- Building with ko for GHCR
+    controller_image_ghcr="$(
+        VERSION="${tag}" \
+        KO_DOCKER_REPO="${controller_repo_ghcr}" \
+            ko build --bare --tags "${version}" --platform linux/amd64,linux/arm64 \
+    )"
+    buildkite-agent meta-data set controller-image-ghcr "${controller_image_ghcr}"
+fi
+
+buildkite-agent annotate --style success --append <<EOF
+### Controller
+----------------------------------------------------------------------------
+| Version    | Image                                                       |
+|------------|-------------------------------------------------------------|
+| ${version} | ${controller_image_pecr:-} <br>  ${controller_image_ghcr:-} |
+----------------------------------------------------------------------------
+EOF

.buildkite/steps/build-and-push-helm.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env ash
+
+set -eufo pipefail
+
+echo --- :hammer: Installing tools
+apk add --update-cache --no-progress aws-cli git helm jq skopeo yq
+
+echo --- :git::docker: determining version and tags
+source .buildkite/steps/repo_info.sh
+
+# TODO: Once the agent is also pushed to PECR, specialise this per registry
+yq -i ".config.image = \"${agent_image}\"" charts/agent-stack-k8s/values.yaml
+
+if [[ "${helm_repo_pecr:-}" != "" ]] ; then
+    source .buildkite/steps/assume-role.sh
+
+    echo "~~~ :helm: Logging into Public ECR"
+    helm registry login public.ecr.aws -u AWS --password "$(aws --region us-east-1 ecr-public get-login-password)"
+
+    echo "--- :helm: Packaging helm chart for Public ECR"
+    yq -i ".image = \"${controller_image_pecr}\"" charts/agent-stack-k8s/values.yaml
+    helm package charts/agent-stack-k8s --app-version "${version}" -d dist --version "${version}"
+
+    echo "--- :helm: Pushing helm chart to public.ecr.aws"
+    helm push "dist/agent-stack-k8s-${version}.tgz" "${helm_repo_pecr}"
+fi
+
+if [[ "${helm_repo_ghcr:-}" != "" ]] ; then
+    echo "--- :docker: Logging into ghcr.io"
+    skopeo login ghcr.io \
+        -u "${REGISTRY_USERNAME}" \
+        --password "${REGISTRY_PASSWORD}" \
+        --authfile ~/.docker/config.json
+
+    echo "--- :helm: Packaging helm chart for GHCR"
+    yq -i ".image = \"${controller_image_ghcr}\"" charts/agent-stack-k8s/values.yaml
+    helm package charts/agent-stack-k8s --app-version "${version}" -d dist --version "${version}"
+
+    echo "--- :helm: Pushing helm chart to ghcr.io"
+    helm push "dist/agent-stack-k8s-${version}.tgz" "${helm_repo_ghcr}"
+fi
+
+buildkite-agent annotate --style success --append <<EOF
+### Helm Chart
+---------------------------------------------------------------
+| Version    | Image                                          |
+|------------|------------------------------------------------|
+| ${version} | ${helm_image_pecr:-} <br> ${helm_image_ghcr:-} |
+---------------------------------------------------------------
+EOF

.buildkite/steps/deploy.sh

@@ -8,7 +8,7 @@ apk add --update-cache --no-progress helm git
 source .buildkite/steps/repo_info.sh
 
 echo --- :helm: Helm upgrade
-helm upgrade agent-stack-k8s "${helm_repo}/agent-stack-k8s" \
+helm upgrade agent-stack-k8s "${helm_repo_pecr}" \
   --version "${version}" \
   --namespace buildkite \
   --install \

.buildkite/steps/release.sh

@@ -16,7 +16,7 @@ GHCH_VERSION="0.11.0"
 GHCH_URL="https://github.com/buildkite/ghch/releases/download/v${GHCH_VERSION}/ghch-$(go env GOARCH)"
 
 echo --- :hammer: Installing packages
-apk add --no-progress crane git
+apk add --no-progress aws-cli crane git
 wget -q "${GORELEASER_URL}/v${GORELEASER_VERSION}/${GORELEASER_FILE}"
 apk add --no-progress --allow-untrusted "${GORELEASER_FILE}"
 rm "${GORELEASER_FILE}"
@@ -36,31 +36,41 @@ echo "Previous tag: ${previous_tag}"
 agent_version="$(buildkite-agent meta-data get agent-version)"
 echo "Agent version: ${agent_version}"
 
+source .buildkite/steps/assume-role.sh
+echo --- Logging into Public ECR
+crane auth login public.ecr.aws \
+  --username AWS \
+  --password "$(aws --region us-east-1 ecr-public get-login-password)"
+
 echo --- :docker: Logging into ghcr.io
 crane auth login ghcr.io \
   --username "${REGISTRY_USERNAME}" \
   --password "${REGISTRY_PASSWORD}"
 
 echo --- :docker: Tagging latest images
+crane tag "public.ecr.aws/buildkite/helm/agent-stack-k8s:${version}" latest
+crane tag "public.ecr.aws/buildkite/agent-stack-k8s/controller:${version}" latest
 crane tag "ghcr.io/buildkite/helm/agent-stack-k8s:${version}" latest
 crane tag "ghcr.io/buildkite/agent-stack-k8s/controller:${version}" latest
 
 echo --- :golang: Creating draft release with goreleaser
-chart_digest=$(crane digest "ghcr.io/buildkite/helm/agent-stack-k8s:$version")
-controller_digest=$(crane digest "ghcr.io/buildkite/agent-stack-k8s/controller:$version")
+chart_digest="$(crane digest "public.ecr.aws/buildkite/helm/agent-stack-k8s:$version")"
+controller_digest="$(crane digest "public.ecr.aws/buildkite/agent-stack-k8s/controller:$version")"
 agent_digest="$(crane digest "ghcr.io/buildkite/agent:${agent_version}")"
 
 # TODO: remove once world write issues is fixed
 git stash -uk
 
-changelog=$(ghch --format=markdown --from="$previous_tag" --next-version="$tag")
+changelog="$(ghch --format=markdown --from="$previous_tag" --next-version="$tag")"
 footer="
 ## Images
 ### Helm chart
+Image: \`public.ecr.aws/buildkite/helm/agent-stack-k8s:${version}\`
 Image: \`ghcr.io/buildkite/helm/agent-stack-k8s:${version}\`
 Digest: \`${chart_digest}\`
 
 ### Controller
+Image: \`public.ecr.aws/buildkite/agent-stack-k8s/controller:${version}\`
 Image: \`ghcr.io/buildkite/agent-stack-k8s/controller:${version}\`
 Digest: \`${controller_digest}\`
 

.buildkite/steps/repo_info.sh

@@ -6,11 +6,30 @@ set -eufo pipefail
 tag="$(git describe)"
 version="${tag#v}"
 agent_image="$(buildkite-agent meta-data get agent-image)"
-controller_image="$(buildkite-agent meta-data get controller-image)"
-helm_repo="oci://ghcr.io/buildkite/helm"
-helm_image="${helm_repo}/agent-stack-k8s:${version}"
+
+# Is this a release version (version-tagged)?
+if [[ "${version}" == "${BUILDKITE_TAG#v}" ]] ; then
+    # Publish releases to both PECR and GHCR
+    controller_repo_pecr="public.ecr.aws/buildkite/agent-stack-k8s/controller"
+    controller_image_pecr="$(buildkite-agent meta-data get controller-image-pecr)"
+    helm_repo_pecr="oci://public.ecr.aws/buildkite/helm"
+    helm_image_pecr="${helm_repo_pecr}/agent-stack-k8s:${version}"
+
+    controller_repo_ghcr="ghcr.io/buildkite/agent-stack-k8s/controller"
+    controller_image_ghcr="$(buildkite-agent meta-data get controller-image-ghcr)"
+    helm_repo_ghcr="oci://ghcr.io/buildkite/helm"
+    helm_image_ghcr="${helm_repo_ghcr}/agent-stack-k8s:${version}"
+else
+    # Publish dev images to PECR dev repo only
+    controller_repo_pecr="public.ecr.aws/buildkite/agent-stack-k8s-dev/controller"
+    controller_image_pecr="$(buildkite-agent meta-data get controller-image-pecr)"
+    helm_repo_pecr="oci://public.ecr.aws/buildkite/helm-dev"
+    helm_image_pecr="${helm_repo_pecr}/agent-stack-k8s:${version}"
+fi
 
 echo "version=${version}"
-echo "controller_image=${controller_image}"
+echo "controller_image_pecr=${controller_image_pecr}"
+echo "controller_image_ghcr=${controller_image_ghcr:-}"
 echo "agent_image=${agent_image}"
-echo "helm_image=${helm_image}"
+echo "helm_image_pecr=${helm_image_pecr}"
+echo "helm_image_ghcr=${helm_image_ghcr:-}"