-
Notifications
You must be signed in to change notification settings - Fork 100
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into bump_alpine
- Loading branch information
Showing
50 changed files
with
649 additions
and
597 deletions.
There are no files selected for viewing
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -e | ||
| set -o pipefail | ||
|
|
||
| # Function to run gcloud commands with the configured project | ||
| gcloudCmd() { | ||
| gcloud --project "$GCP_PROJECT" "$@" | ||
| } | ||
|
|
||
| # Function to get the highest 4 stable versions from GCE images | ||
| getHighest4StableVersions() { | ||
| local versions | ||
| local stableVersions=() | ||
| local sortedVersions | ||
| local highest4StableVersions | ||
|
|
||
| # Get all Kairos image versions | ||
| mapfile -t versions < <(gcloudCmd compute images list --filter="family=kairos" --format="value(labels.version)" | sort -u) | ||
|
|
||
| # Filter out non-stable versions (those containing '-rc') | ||
| for version in "${versions[@]}"; do | ||
| if [[ ! $version =~ -rc ]]; then | ||
| stableVersions+=("$version") | ||
| fi | ||
| done | ||
|
|
||
| # Sort versions and get the highest 4 | ||
| IFS=$'\n' mapfile -t sortedVersions < <(printf '%s\n' "${stableVersions[@]}" | sort -V -r) | ||
| unset IFS | ||
| highest4StableVersions=("${sortedVersions[@]:0:4}") | ||
|
|
||
| # Return the highest 4 stable versions | ||
| echo "${highest4StableVersions[@]}" | ||
| } | ||
|
|
||
| # Function to delete images that are not in the latest 4 stable versions | ||
| imageDeleteIfNotInVersionList() { | ||
| local image=$1 | ||
| shift 1 | ||
| local versionList=("$@") | ||
|
|
||
| # Get the image version | ||
| local imageVersion | ||
| imageVersion=$(gcloudCmd compute images describe "$image" --format="value(labels.version)" 2>/dev/null || echo "UNKNOWN") | ||
|
|
||
| # Check if imageVersion is in versionList | ||
| local versionFound="false" | ||
| for version in "${versionList[@]}"; do | ||
| if [[ "$imageVersion" == "$version" ]]; then | ||
| versionFound="true" | ||
| break | ||
| fi | ||
| done | ||
|
|
||
| if [[ "$versionFound" == "true" ]]; then | ||
| echo "Image $image has a stable version ($imageVersion). Skipping cleanup." | ||
| else | ||
| gcloudCmd compute images delete "$image" --quiet | ||
| echo "Deleted image $image as it does not match any of the versions: '${versionList[*]}'" | ||
| fi | ||
| } | ||
|
|
||
| # Function to clean up old GCS objects | ||
| s3ObjectDeleteIfNotInVersionList() { | ||
| local key=$1 | ||
| shift 1 | ||
| local versionList=("$@") | ||
|
|
||
| # Get object tags (labels in GCS) | ||
| local objectVersion | ||
| objectVersion=$(gcloudCmd storage objects describe "$key" --format="value(custom_fields.version)" 2>/dev/null || echo "UNKNOWN") | ||
|
|
||
| # Check if objectVersion is in versionList | ||
| local versionFound="false" | ||
| for version in "${versionList[@]}"; do | ||
| if [[ "$objectVersion" == "$version" ]]; then | ||
| versionFound="true" | ||
| break | ||
| fi | ||
| done | ||
|
|
||
| if [[ "$versionFound" == "true" ]]; then | ||
| echo "GCS object '$key' has a stable version ($objectVersion). Skipping cleanup." | ||
| else | ||
| gcloudCmd storage rm "$key" | ||
| echo "Deleted GCS object $key as it does not match any of the versions: '${versionList[*]}'" | ||
| fi | ||
| } | ||
|
|
||
| # Main cleanup function | ||
| cleanupOldVersions() { | ||
| if [ -z "$GCP_PROJECT" ] || [ -z "$GCS_BUCKET" ]; then | ||
| echo "Error: GCP_PROJECT and GCS_BUCKET environment variables must be set." | ||
| exit 1 | ||
| fi | ||
|
|
||
| read -r -a highest4StableVersions < <(getHighest4StableVersions) | ||
| echo "Highest 4 stable versions: ${highest4StableVersions[*]}" | ||
|
|
||
| # Cleanup images | ||
| mapfile -t allImages < <(gcloudCmd compute images list --filter="family=kairos" --format="value(name)") | ||
| for img in "${allImages[@]}"; do | ||
| imageDeleteIfNotInVersionList "$img" "${highest4StableVersions[@]}" | ||
| done | ||
|
|
||
| # Cleanup GCS objects | ||
| mapfile -t allS3Objects < <(gcloudCmd storage ls "gs://$GCS_BUCKET") | ||
| for s3Object in "${allS3Objects[@]}"; do | ||
| s3ObjectDeleteIfNotInVersionList "$s3Object" "${highest4StableVersions[@]}" | ||
| done | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| #!/bin/bash | ||
|
|
||
| # This script uploads a raw disk image to Google Cloud Storage, | ||
| # imports it as a GCE image, makes it public, and replicates it across regions. | ||
| # Equivalent to AWS AMI upload script. | ||
|
|
||
| set -e | ||
| set -o pipefail | ||
| set -x | ||
|
|
||
| SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | ||
| # shellcheck source=/dev/null | ||
| source "$SCRIPT_DIR/cleanup-old-images-gce.sh" | ||
|
|
||
| checkArguments() { | ||
| if [ $# -lt 2 ]; then | ||
| echo "Error: You need to specify the cloud image to upload and the Kairos version (to tag resources)." | ||
| echo "Usage: $0 <cloud-image> <kairos-version>" | ||
| exit 1 | ||
| fi | ||
|
|
||
| local file="$1" | ||
|
|
||
| if [ ! -f "$file" ]; then | ||
| echo "Error: File '$file' does not exist." | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| checkEnvVars() { | ||
| if [ -z "$GCP_PROJECT" ] || [ -z "$GCS_BUCKET" ]; then | ||
| echo "Error: GCP_PROJECT and GCS_BUCKET environment variables must be set." | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| gcloudCmd() { | ||
| gcloud --project "$GCP_PROJECT" "$@" | ||
| } | ||
|
|
||
| uploadImageToGCS() { | ||
| local file="$1" | ||
| local name="$2" | ||
| local version="$3" | ||
|
|
||
| if gsutil ls "gs://$GCS_BUCKET/$name" > /dev/null 2>&1; then | ||
| echo "File '$name' already exists in GCS bucket '$GCS_BUCKET'." | ||
| else | ||
| echo "Uploading '$name' to GCS bucket '$GCS_BUCKET'." | ||
| gsutil -h "x-goog-meta-version:$version" cp "$file" "gs://$GCS_BUCKET/$name" | ||
| fi | ||
| } | ||
|
|
||
| importGceImage() { | ||
| local fileName="$1" | ||
| local name | ||
| local kairosVersion="$2" | ||
| name="$(sanitizeString "$fileName")" | ||
|
|
||
| if gcloud compute images describe "$name" --project "$GCP_PROJECT" > /dev/null 2>&1; then | ||
| echo "Image '$name' already exists." | ||
|
|
||
| echo "Making sure it has the version label" | ||
| gcloud compute images add-labels "$name" \ | ||
| --project "$GCP_PROJECT" \ | ||
| --labels="version=$kairosVersion" | ||
| else | ||
| echo "Importing image '$name' from GCS." | ||
| gcloud migration vms image-imports create "$name" \ | ||
| --image-name="$name" \ | ||
| --location=europe-west3 \ | ||
| --target-project="$GCP_PROJECT" \ | ||
| --source-file="gs://$GCS_BUCKET/$fileName" \ | ||
| --family-name="kairos" \ | ||
| --skip-os-adaptation \ | ||
| --labels="version=$kairosVersion" | ||
|
|
||
| while true; do | ||
| status=$(gcloud compute images describe "$name" --project="$GCP_PROJECT" --format="value(status)" 2>/dev/null || echo "NOT_FOUND") | ||
|
|
||
| if [[ "$status" == "READY" ]]; then | ||
| echo "Import completed successfully!" | ||
| break | ||
| elif [[ "$status" == "FAILED" ]]; then | ||
| echo "Import failed!" | ||
| exit 1 | ||
| elif [[ "$status" == "NOT_FOUND" ]]; then | ||
| echo "Image not found yet, waiting..." | ||
| else | ||
| echo "Still in progress... (Current status: $status)" | ||
| fi | ||
|
|
||
| sleep 30 # Wait before checking again | ||
| done | ||
| fi | ||
|
|
||
| echo "$name" | ||
|
|
||
| # https://cloud.google.com/compute/docs/images/managing-access-custom-images | ||
| # Make the image public | ||
| gcloud compute images add-iam-policy-binding "$name" \ | ||
| --member='allAuthenticatedUsers' \ | ||
| --role='roles/compute.imageUser' | ||
| echo "Image '$name' is now public." | ||
|
|
||
| # Cleanup: delete the image import after the image is imported | ||
| echo "Cleaning up by deleting the image import process." | ||
| gcloud migration vms image-imports delete "$name" \ | ||
| --location=europe-west3 \ | ||
| --project="$GCP_PROJECT" \ | ||
| --quiet | ||
| echo "Import process for '$name' has been deleted." | ||
| } | ||
|
|
||
| # Sanitize names by replacing "." with "-" and removing extensions | ||
| # Can be used to generate valid resource names or labels from file paths or | ||
| # versions. | ||
| sanitizeString() { | ||
| local baseName | ||
| baseName=$(basename "$1") | ||
| echo "${baseName%.tar.gz}" | tr '.' '-' | ||
| } | ||
|
|
||
| # ----- Main script ----- | ||
| imageFile="$1" | ||
| kairosVersion=$(sanitizeString "$2") | ||
| checkEnvVars | ||
| checkArguments "$@" | ||
|
|
||
| echo | ||
| echo "Performing cleanup of old versions" | ||
| cleanupOldVersions | ||
| echo "Done cleaning up" | ||
| echo | ||
|
|
||
| name=$(sanitizeString "$imageFile") | ||
| fileName=$(basename "$imageFile") | ||
|
|
||
| # Note: It's likely that we can point --source-file to the local file directly when we | ||
| # import the image. This would allow us to skip uploading to the bucket. We do it like this, | ||
| # to be able to check the used raw image file in case the image doesn't work for whatever reason. | ||
| # Cleanup only keeps 5 files around so it shouldn't cost much. | ||
| uploadImageToGCS "$imageFile" "$fileName" "$kairosVersion" | ||
| importGceImage "$fileName" "$kairosVersion" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| name: Build Examples | ||
|
|
||
| on: | ||
| push: | ||
| paths: | ||
| - 'examples/builds/**' | ||
|
|
||
| jobs: | ||
| build: | ||
| name: ${{ matrix.example-dir }} | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| matrix: | ||
| example-dir: | ||
| - "fedora-fips" | ||
| - "rockylinux-fips" | ||
| - "ubuntu-non-hwe" | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Build Dockerfile | ||
| uses: docker/build-push-action@v6 | ||
| with: | ||
| file: examples/builds/${{ matrix.example-dir }}/Dockerfile | ||
| context: examples/builds/${{ matrix.example-dir }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -74,7 +74,7 @@ jobs: | |
| LUET_NOLOCK=true sudo -E luet install -y container/kubectl utils/k3d | ||
| - name: Download ISO | ||
| id: iso | ||
| uses: actions/[email protected].8 | ||
| uses: actions/[email protected].9 | ||
| with: | ||
| name: kairos-${{ steps.split.outputs.flavor }}-${{ steps.split.outputs.flavor_release }}-${{ inputs.variant }}-${{ inputs.arch }}-${{ inputs.model }}.iso.zip | ||
| - name: Display structure of downloaded files | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -75,7 +75,7 @@ jobs: | |
| cp ipxe/src/bin/ipxe.iso ipxe.iso | ||
| - name: Download ISO | ||
| id: iso | ||
| uses: actions/[email protected].8 | ||
| uses: actions/[email protected].9 | ||
| with: | ||
| name: kairos-${{ steps.split.outputs.flavor }}-${{ steps.split.outputs.flavor_release }}-${{ inputs.variant }}-${{ inputs.arch }}-${{ inputs.model }}.iso.zip | ||
| - name: Extract ipxe artifacts for the ipxe server | ||
|
|
||
Oops, something went wrong.