From 6fe352f565c682ddfa2a9a466a50c16ddb9192fc Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Wed, 23 Apr 2025 19:59:32 +0100 Subject: [PATCH 01/30] feat: Dockerfile containing all dependencies for tasks Signed-off-by: Evans Mungai --- applications/wg-easy/Dockerfile | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 applications/wg-easy/Dockerfile diff --git a/applications/wg-easy/Dockerfile b/applications/wg-easy/Dockerfile new file mode 100644 index 00000000..a106233d --- /dev/null +++ b/applications/wg-easy/Dockerfile @@ -0,0 +1,60 @@ +FROM --platform=$BUILDPLATFORM ubuntu:24.04 AS build + +ARG TARGETOS +ARG TARGETARCH + +# Tool versions +ARG REPLICATED_VERSION=0.101.1 +ARG TASK_VERSION=3.43.2 +ARG KUBECTL_VERSION=1.30.1 +ARG HELM_VERSION=3.17.3 +ARG YQ_VERSION=4.45.1 + +RUN apt-get update && apt-get install -y \ + curl \ + jq \ + yq \ + gpg \ + && rm -rf /var/lib/apt/lists/* + +# Install the Google Cloud tools following the instructions here: https://cloud.google.com/sdk/docs/install#deb +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ + && apt-get update \ + && apt-get install google-cloud-cli -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tools + +# TODO: use checksums to verify the downloaded files + +# TODO: Use this to build for different platforms +RUN echo "=======================================================" +RUN echo "Building for TARGET PLATFORM: ${TARGETOS}_${TARGETARCH}" +RUN echo "=======================================================" + +# Download and extract the replicated binary +RUN curl -L -o replicated.tar.gz https://github.com/replicatedhq/replicated/releases/download/v${REPLICATED_VERSION}/replicated_${REPLICATED_VERSION}_linux_amd64.tar.gz \ + && tar -xzf replicated.tar.gz \ + && rm -f replicated.tar.gz LICENSE README.md + +# Download and extract the task binary +RUN curl -L -o task.tar.gz https://github.com/go-task/task/releases/download/v${TASK_VERSION}/task_${TARGETOS}_${TARGETARCH}.tar.gz \ + && tar -xzf task.tar.gz \ + && rm -rf task.tar.gz LICENSE README.md completion/ + +# Download and extract the kubectl binary +RUN curl -L -o kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${TARGETOS}/${TARGETARCH}/kubectl \ + && chmod +x kubectl + +# Download and extract the helm binary +RUN curl -L -o helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz \ + && tar -xzf helm.tar.gz \ + && mv ${TARGETOS}-${TARGETARCH}/helm . \ + && rm -rf ${TARGETOS}-${TARGETARCH} + +# Download and extract the yq binary +RUN curl -L -o yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${TARGETOS}_${TARGETARCH} \ + && chmod +x yq + +ENV PATH="/tools:${PATH}" From 48d112bffc6baedf4d236cc7a6d3eeb698fe64a3 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Thu, 24 Apr 2025 12:58:06 +0100 Subject: [PATCH 02/30] download helmfile Signed-off-by: Evans Mungai --- applications/wg-easy/Dockerfile | 10 +++++++++- applications/wg-easy/Taskfile.yaml | 32 +++++++++++++++--------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/applications/wg-easy/Dockerfile b/applications/wg-easy/Dockerfile index a106233d..950a9d66 100644 --- a/applications/wg-easy/Dockerfile +++ b/applications/wg-easy/Dockerfile @@ -8,6 +8,7 @@ ARG REPLICATED_VERSION=0.101.1 ARG TASK_VERSION=3.43.2 ARG KUBECTL_VERSION=1.30.1 ARG HELM_VERSION=3.17.3 +ARG HELMFILE_VERSION=0.171.0 ARG YQ_VERSION=4.45.1 RUN apt-get update && apt-get install -y \ @@ -51,10 +52,17 @@ RUN curl -L -o kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${TARGE RUN curl -L -o helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz \ && tar -xzf helm.tar.gz \ && mv ${TARGETOS}-${TARGETARCH}/helm . \ - && rm -rf ${TARGETOS}-${TARGETARCH} + && rm -rf ${TARGETOS}-${TARGETARCH} helm.tar.gz + +# Download helmfile binary +RUN curl -vL -o helmfile.tar.gz https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz \ + && tar -xzf helmfile.tar.gz \ + && rm -f helmfile.tar.gz # Download and extract the yq binary RUN curl -L -o yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${TARGETOS}_${TARGETARCH} \ && chmod +x yq +WORKDIR /app + ENV PATH="/tools:${PATH}" diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index ae079c6e..91c2525d 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -6,7 +6,7 @@ includes: vars: # Application configuration APP_NAME: '{{.REPLICATED_APP | default "wg-easy"}}' - + # Cluster configuration CLUSTER_NAME: '{{.CLUSTER_NAME | default "test-cluster"}}' K8S_VERSION: '{{.K8S_VERSION | default "1.32.2"}}' @@ -14,14 +14,14 @@ vars: INSTANCE_TYPE: '{{.INSTANCE_TYPE | default "r1.small"}}' DISTRIBUTION: '{{.DISTRIBUTION | default "k3s"}}' KUBECONFIG_FILE: './{{.CLUSTER_NAME}}.kubeconfig' - + # Ports configuration EXPOSE_PORTS: - port: 30443 protocol: https - port: 30080 protocol: http - + # GCP default configuration GCP_PROJECT: '{{.GCP_PROJECT | default "replicated-qa"}}' GCP_ZONE: '{{.GCP_ZONE | default "us-central1-a"}}' @@ -91,7 +91,7 @@ tasks: echo "Removing old kubeconfig file" rm -f {{.KUBECONFIG_FILE}} fi - fi + fi setup-kubeconfig: desc: Get kubeconfig and prepare cluster for application deployment @@ -112,7 +112,7 @@ tasks: true fi deps: - - create-cluster + - cluster-create - verify-kubeconfig dependencies-update: @@ -138,7 +138,7 @@ tasks: if [ -z "$CLUSTER_ID" ]; then exit 1 fi - + # Check if all ports are already exposed expected_count={{len .EXPOSE_PORTS}} port_checks="" @@ -147,7 +147,7 @@ tasks: {{end}} # Remove trailing "or " port_checks="${port_checks% or }" - + PORT_COUNT=$(replicated cluster port ls $CLUSTER_ID --output json | jq -r ".[] | select($port_checks) | .upstream_port" | wc -l | tr -d ' ') [ "$PORT_COUNT" -eq "$expected_count" ] cmds: @@ -169,10 +169,10 @@ tasks: echo "Error: Could not find cluster with name {{.CLUSTER_NAME}}" exit 1 fi - + # Get exposed URLs ENV_VARS=$(task utils:port-operations OPERATION=getenv CLUSTER_NAME={{.CLUSTER_NAME}}) - + # Deploy with helmfile echo "Using $ENV_VARS" eval "KUBECONFIG={{.KUBECONFIG_FILE}} $ENV_VARS helmfile sync --wait" @@ -193,7 +193,7 @@ tasks: echo "No clusters found with name {{.CLUSTER_NAME}}" exit 0 fi - + for id in $CLUSTER_IDS; do echo "Deleting cluster ID: $id" replicated cluster rm "$id" @@ -213,7 +213,7 @@ tasks: - echo "Preparing release files..." - rm -rf ./release - mkdir -p ./release - + # Copy all non-config.yaml files - echo "Copying non-config YAML files to release folder..." - find . -path '*/replicated/*.yaml' -not -name 'config.yaml' -exec cp {} ./release/ \; @@ -237,27 +237,27 @@ tasks: yq '.spec.chart.chartVersion = strenv(version) | .spec.chart.chartVersion style="single"' $directory/$helmChartName | tee release/$helmChartName done < <(find . -maxdepth 2 -mindepth 2 -type d -name replicated) - + # Merge config.yaml files - echo "Merging config.yaml files..." - | # Start with an empty config file echo "{}" > ./release/config.yaml - + # Merge all app config.yaml files first (excluding root replicated) for config_file in $(find . -path '*/replicated/config.yaml' | grep -v "^./replicated/"); do echo "Merging $config_file..." yq eval-all '. as $item ireduce ({}; . * $item)' ./release/config.yaml "$config_file" > ./release/config.yaml.new mv ./release/config.yaml.new ./release/config.yaml done - + # Merge root config.yaml last if [ -f "./replicated/config.yaml" ]; then echo "Merging root config.yaml last..." yq eval-all '. as $item ireduce ({}; . * $item)' ./release/config.yaml "./replicated/config.yaml" > ./release/config.yaml.new mv ./release/config.yaml.new ./release/config.yaml fi - + # Package Helm charts - echo "Packaging Helm charts..." - | @@ -267,7 +267,7 @@ tasks: # Navigate to chart directory, package it, and move the resulting .tgz to release folder (cd "$chart_dir" && helm package . && mv *.tgz ../release/) done - + - echo "Release files prepared in ./release/ directory" deps: - update-version From b26044debaa23e24bd792edb0cd7c2bc07bef246 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Wed, 23 Apr 2025 19:59:32 +0100 Subject: [PATCH 03/30] feat: Dockerfile containing all dependencies for tasks Signed-off-by: Evans Mungai --- applications/wg-easy/Dockerfile | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 applications/wg-easy/Dockerfile diff --git a/applications/wg-easy/Dockerfile b/applications/wg-easy/Dockerfile new file mode 100644 index 00000000..a106233d --- /dev/null +++ b/applications/wg-easy/Dockerfile @@ -0,0 +1,60 @@ +FROM --platform=$BUILDPLATFORM ubuntu:24.04 AS build + +ARG TARGETOS +ARG TARGETARCH + +# Tool versions +ARG REPLICATED_VERSION=0.101.1 +ARG TASK_VERSION=3.43.2 +ARG KUBECTL_VERSION=1.30.1 +ARG HELM_VERSION=3.17.3 +ARG YQ_VERSION=4.45.1 + +RUN apt-get update && apt-get install -y \ + curl \ + jq \ + yq \ + gpg \ + && rm -rf /var/lib/apt/lists/* + +# Install the Google Cloud tools following the instructions here: https://cloud.google.com/sdk/docs/install#deb +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ + && apt-get update \ + && apt-get install google-cloud-cli -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /tools + +# TODO: use checksums to verify the downloaded files + +# TODO: Use this to build for different platforms +RUN echo "=======================================================" +RUN echo "Building for TARGET PLATFORM: ${TARGETOS}_${TARGETARCH}" +RUN echo "=======================================================" + +# Download and extract the replicated binary +RUN curl -L -o replicated.tar.gz https://github.com/replicatedhq/replicated/releases/download/v${REPLICATED_VERSION}/replicated_${REPLICATED_VERSION}_linux_amd64.tar.gz \ + && tar -xzf replicated.tar.gz \ + && rm -f replicated.tar.gz LICENSE README.md + +# Download and extract the task binary +RUN curl -L -o task.tar.gz https://github.com/go-task/task/releases/download/v${TASK_VERSION}/task_${TARGETOS}_${TARGETARCH}.tar.gz \ + && tar -xzf task.tar.gz \ + && rm -rf task.tar.gz LICENSE README.md completion/ + +# Download and extract the kubectl binary +RUN curl -L -o kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${TARGETOS}/${TARGETARCH}/kubectl \ + && chmod +x kubectl + +# Download and extract the helm binary +RUN curl -L -o helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz \ + && tar -xzf helm.tar.gz \ + && mv ${TARGETOS}-${TARGETARCH}/helm . \ + && rm -rf ${TARGETOS}-${TARGETARCH} + +# Download and extract the yq binary +RUN curl -L -o yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${TARGETOS}_${TARGETARCH} \ + && chmod +x yq + +ENV PATH="/tools:${PATH}" From 2ef46337b6663783025e61da952c841690cee0ce Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Thu, 24 Apr 2025 12:58:06 +0100 Subject: [PATCH 04/30] download helmfile Signed-off-by: Evans Mungai --- applications/wg-easy/Dockerfile | 10 +++++++++- applications/wg-easy/Taskfile.yaml | 32 +++++++++++++++--------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/applications/wg-easy/Dockerfile b/applications/wg-easy/Dockerfile index a106233d..950a9d66 100644 --- a/applications/wg-easy/Dockerfile +++ b/applications/wg-easy/Dockerfile @@ -8,6 +8,7 @@ ARG REPLICATED_VERSION=0.101.1 ARG TASK_VERSION=3.43.2 ARG KUBECTL_VERSION=1.30.1 ARG HELM_VERSION=3.17.3 +ARG HELMFILE_VERSION=0.171.0 ARG YQ_VERSION=4.45.1 RUN apt-get update && apt-get install -y \ @@ -51,10 +52,17 @@ RUN curl -L -o kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${TARGE RUN curl -L -o helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz \ && tar -xzf helm.tar.gz \ && mv ${TARGETOS}-${TARGETARCH}/helm . \ - && rm -rf ${TARGETOS}-${TARGETARCH} + && rm -rf ${TARGETOS}-${TARGETARCH} helm.tar.gz + +# Download helmfile binary +RUN curl -vL -o helmfile.tar.gz https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz \ + && tar -xzf helmfile.tar.gz \ + && rm -f helmfile.tar.gz # Download and extract the yq binary RUN curl -L -o yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${TARGETOS}_${TARGETARCH} \ && chmod +x yq +WORKDIR /app + ENV PATH="/tools:${PATH}" diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index ae079c6e..91c2525d 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -6,7 +6,7 @@ includes: vars: # Application configuration APP_NAME: '{{.REPLICATED_APP | default "wg-easy"}}' - + # Cluster configuration CLUSTER_NAME: '{{.CLUSTER_NAME | default "test-cluster"}}' K8S_VERSION: '{{.K8S_VERSION | default "1.32.2"}}' @@ -14,14 +14,14 @@ vars: INSTANCE_TYPE: '{{.INSTANCE_TYPE | default "r1.small"}}' DISTRIBUTION: '{{.DISTRIBUTION | default "k3s"}}' KUBECONFIG_FILE: './{{.CLUSTER_NAME}}.kubeconfig' - + # Ports configuration EXPOSE_PORTS: - port: 30443 protocol: https - port: 30080 protocol: http - + # GCP default configuration GCP_PROJECT: '{{.GCP_PROJECT | default "replicated-qa"}}' GCP_ZONE: '{{.GCP_ZONE | default "us-central1-a"}}' @@ -91,7 +91,7 @@ tasks: echo "Removing old kubeconfig file" rm -f {{.KUBECONFIG_FILE}} fi - fi + fi setup-kubeconfig: desc: Get kubeconfig and prepare cluster for application deployment @@ -112,7 +112,7 @@ tasks: true fi deps: - - create-cluster + - cluster-create - verify-kubeconfig dependencies-update: @@ -138,7 +138,7 @@ tasks: if [ -z "$CLUSTER_ID" ]; then exit 1 fi - + # Check if all ports are already exposed expected_count={{len .EXPOSE_PORTS}} port_checks="" @@ -147,7 +147,7 @@ tasks: {{end}} # Remove trailing "or " port_checks="${port_checks% or }" - + PORT_COUNT=$(replicated cluster port ls $CLUSTER_ID --output json | jq -r ".[] | select($port_checks) | .upstream_port" | wc -l | tr -d ' ') [ "$PORT_COUNT" -eq "$expected_count" ] cmds: @@ -169,10 +169,10 @@ tasks: echo "Error: Could not find cluster with name {{.CLUSTER_NAME}}" exit 1 fi - + # Get exposed URLs ENV_VARS=$(task utils:port-operations OPERATION=getenv CLUSTER_NAME={{.CLUSTER_NAME}}) - + # Deploy with helmfile echo "Using $ENV_VARS" eval "KUBECONFIG={{.KUBECONFIG_FILE}} $ENV_VARS helmfile sync --wait" @@ -193,7 +193,7 @@ tasks: echo "No clusters found with name {{.CLUSTER_NAME}}" exit 0 fi - + for id in $CLUSTER_IDS; do echo "Deleting cluster ID: $id" replicated cluster rm "$id" @@ -213,7 +213,7 @@ tasks: - echo "Preparing release files..." - rm -rf ./release - mkdir -p ./release - + # Copy all non-config.yaml files - echo "Copying non-config YAML files to release folder..." - find . -path '*/replicated/*.yaml' -not -name 'config.yaml' -exec cp {} ./release/ \; @@ -237,27 +237,27 @@ tasks: yq '.spec.chart.chartVersion = strenv(version) | .spec.chart.chartVersion style="single"' $directory/$helmChartName | tee release/$helmChartName done < <(find . -maxdepth 2 -mindepth 2 -type d -name replicated) - + # Merge config.yaml files - echo "Merging config.yaml files..." - | # Start with an empty config file echo "{}" > ./release/config.yaml - + # Merge all app config.yaml files first (excluding root replicated) for config_file in $(find . -path '*/replicated/config.yaml' | grep -v "^./replicated/"); do echo "Merging $config_file..." yq eval-all '. as $item ireduce ({}; . * $item)' ./release/config.yaml "$config_file" > ./release/config.yaml.new mv ./release/config.yaml.new ./release/config.yaml done - + # Merge root config.yaml last if [ -f "./replicated/config.yaml" ]; then echo "Merging root config.yaml last..." yq eval-all '. as $item ireduce ({}; . * $item)' ./release/config.yaml "./replicated/config.yaml" > ./release/config.yaml.new mv ./release/config.yaml.new ./release/config.yaml fi - + # Package Helm charts - echo "Packaging Helm charts..." - | @@ -267,7 +267,7 @@ tasks: # Navigate to chart directory, package it, and move the resulting .tgz to release folder (cd "$chart_dir" && helm package . && mv *.tgz ../release/) done - + - echo "Release files prepared in ./release/ directory" deps: - update-version From dfe10c3b3b2bca21099960360b489b224ec0bac7 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 14:20:48 +0100 Subject: [PATCH 05/30] Add initial docker env taskfile Signed-off-by: Evans Mungai --- applications/taskfiles/docker.yml | 120 +++++++++++++++++++++++++++++ applications/wg-easy/Taskfile.yaml | 6 ++ 2 files changed, 126 insertions(+) create mode 100644 applications/taskfiles/docker.yml diff --git a/applications/taskfiles/docker.yml b/applications/taskfiles/docker.yml new file mode 100644 index 00000000..acc10ea4 --- /dev/null +++ b/applications/taskfiles/docker.yml @@ -0,0 +1,120 @@ +version: "3" + +# Development environment tasks +tasks: + build-image: + desc: Build development Docker image + vars: + IMAGE_NAME: '{{.IMAGE_NAME}}' + IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' + DOCKERFILE: '{{.DOCKERFILE | default "Dockerfile"}}' + DOCKERFILE_PATH: '{{.DOCKERFILE_PATH | default "."}}' + BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' + requires: + vars: [IMAGE_NAME, IMAGE_TAG, DOCKERFILE, DOCKERFILE_PATH] + + cmds: + - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} {{.DOCKERFILE_PATH}} + + start: + desc: Start development container in background + silent: true + vars: + IMAGE_NAME: '{{.IMAGE_NAME}}' + CONTAINER_NAME: '{{.CONTAINER_NAME}}' + IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' + requires: + vars: [IMAGE_NAME, CONTAINER_NAME] + + cmds: + - echo "Starting {{.CONTAINER_NAME}} development container..." + - | + # Check if container is running + if docker ps | grep -q "{{.CONTAINER_NAME}}"; then + echo "Container {{.CONTAINER_NAME}} is already running" + # Check if container exists but is not running + elif docker ps -a | grep -q "{{.CONTAINER_NAME}}"; then + echo "Container {{.CONTAINER_NAME}} exists but is not running. Starting it..." + docker start {{.CONTAINER_NAME}} + # Create and start new container + else + # Try ports 5000-5004, use the first available one + PORT_FOUND=false + for PORT in {5000..5004}; do + if ! lsof -i :$PORT > /dev/null; then + PORT_ARG="-p $PORT:$PORT" + echo "Using port $PORT for MLflow UI" + PORT_FOUND=true + break + fi + done + + # If all standard ports are in use, use a random port + if [ "$PORT_FOUND" = "false" ]; then + PORT_ARG="-P" + echo "All standard ports are in use. Using a random port." + fi + + # Start container with host networking for kubectl port-forward compatibility + CONTAINER_ID=$(docker run --name {{.CONTAINER_NAME}} --network host -d \ + -v $(pwd):/workspace \ + -v ~/.kube:/home/devuser/.kube \ + -v ~/.helm:/home/devuser/.helm \ + -v ~/.replicated:/home/devuser/.replicated \ + -e SHELL=/bin/bash \ + -e HOME=/home/devuser \ + -e USER=devuser \ + -e HOST_NETWORK=true \ + -w /workspace \ + {{.IMAGE_NAME}}:{{.IMAGE_TAG}} sleep infinity) + + if [ $? -eq 0 ]; then + echo "Development container started successfully with ID: $CONTAINER_ID" + echo "Ports opened within the container will be directly accessible on your host machine." + else + echo "Failed to start development container" + exit 1 + fi + fi + + ensure-running: + desc: Ensure the development container is running + internal: true + silent: true + cmds: + - | + if ! docker ps | grep -q "{{.CONTAINER_NAME}}"; then + echo "Container {{.CONTAINER_NAME}} is not running. Starting it..." + task start + else + echo "Container {{.CONTAINER_NAME}} is already running." + fi + status: + - docker ps | grep -q "{{.CONTAINER_NAME}}" + + shell: + desc: Attach to development container shell + deps: [ensure-running] + silent: true + cmds: + - echo "Connecting to {{.CONTAINER_NAME}}..." + - docker exec -it {{.CONTAINER_NAME}} /bin/bash + + stop: + desc: Stop development container + silent: true + cmds: + - | + if docker ps | grep -q "{{.CONTAINER_NAME}}"; then + echo "Stopping {{.CONTAINER_NAME}} development container..." + docker stop {{.CONTAINER_NAME}} + else + echo "Container {{.CONTAINER_NAME}} is not running" + fi + + restart: + desc: Restart development container + silent: true + cmds: + - task: stop + - task: start diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index 91c2525d..861775c6 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -2,6 +2,7 @@ version: "3" includes: utils: ./taskfiles/utils.yml + dev: ./../taskfiles/docker.yml vars: # Application configuration @@ -27,6 +28,11 @@ vars: GCP_ZONE: '{{.GCP_ZONE | default "us-central1-a"}}' VM_NAME: '{{.VM_NAME | default (printf "%s-dev" (or (env "GUSER") "user"))}}' + # Docker workflow configuration + IMAGE_NAME: ttl.sh/wg-easy-dev + CONTAINER_NAME: wg-easy-dev + IMAGE_TAG: latest + tasks: default: desc: Show available tasks From 30c6a4c03ff870972f9209aae85a7246049233c2 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 18:54:30 +0100 Subject: [PATCH 06/30] Move docker tasks into wg-easy application Signed-off-by: Evans Mungai --- applications/taskfiles/docker.yml | 120 ------------------- applications/wg-easy/Dockerfile | 68 ----------- applications/wg-easy/Taskfile.yaml | 3 +- applications/wg-easy/container/Containerfile | 67 +++++++++++ applications/wg-easy/taskfiles/container.yml | 87 ++++++++++++++ 5 files changed, 155 insertions(+), 190 deletions(-) delete mode 100644 applications/taskfiles/docker.yml delete mode 100644 applications/wg-easy/Dockerfile create mode 100644 applications/wg-easy/container/Containerfile create mode 100644 applications/wg-easy/taskfiles/container.yml diff --git a/applications/taskfiles/docker.yml b/applications/taskfiles/docker.yml deleted file mode 100644 index acc10ea4..00000000 --- a/applications/taskfiles/docker.yml +++ /dev/null @@ -1,120 +0,0 @@ -version: "3" - -# Development environment tasks -tasks: - build-image: - desc: Build development Docker image - vars: - IMAGE_NAME: '{{.IMAGE_NAME}}' - IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' - DOCKERFILE: '{{.DOCKERFILE | default "Dockerfile"}}' - DOCKERFILE_PATH: '{{.DOCKERFILE_PATH | default "."}}' - BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' - requires: - vars: [IMAGE_NAME, IMAGE_TAG, DOCKERFILE, DOCKERFILE_PATH] - - cmds: - - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} {{.DOCKERFILE_PATH}} - - start: - desc: Start development container in background - silent: true - vars: - IMAGE_NAME: '{{.IMAGE_NAME}}' - CONTAINER_NAME: '{{.CONTAINER_NAME}}' - IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' - requires: - vars: [IMAGE_NAME, CONTAINER_NAME] - - cmds: - - echo "Starting {{.CONTAINER_NAME}} development container..." - - | - # Check if container is running - if docker ps | grep -q "{{.CONTAINER_NAME}}"; then - echo "Container {{.CONTAINER_NAME}} is already running" - # Check if container exists but is not running - elif docker ps -a | grep -q "{{.CONTAINER_NAME}}"; then - echo "Container {{.CONTAINER_NAME}} exists but is not running. Starting it..." - docker start {{.CONTAINER_NAME}} - # Create and start new container - else - # Try ports 5000-5004, use the first available one - PORT_FOUND=false - for PORT in {5000..5004}; do - if ! lsof -i :$PORT > /dev/null; then - PORT_ARG="-p $PORT:$PORT" - echo "Using port $PORT for MLflow UI" - PORT_FOUND=true - break - fi - done - - # If all standard ports are in use, use a random port - if [ "$PORT_FOUND" = "false" ]; then - PORT_ARG="-P" - echo "All standard ports are in use. Using a random port." - fi - - # Start container with host networking for kubectl port-forward compatibility - CONTAINER_ID=$(docker run --name {{.CONTAINER_NAME}} --network host -d \ - -v $(pwd):/workspace \ - -v ~/.kube:/home/devuser/.kube \ - -v ~/.helm:/home/devuser/.helm \ - -v ~/.replicated:/home/devuser/.replicated \ - -e SHELL=/bin/bash \ - -e HOME=/home/devuser \ - -e USER=devuser \ - -e HOST_NETWORK=true \ - -w /workspace \ - {{.IMAGE_NAME}}:{{.IMAGE_TAG}} sleep infinity) - - if [ $? -eq 0 ]; then - echo "Development container started successfully with ID: $CONTAINER_ID" - echo "Ports opened within the container will be directly accessible on your host machine." - else - echo "Failed to start development container" - exit 1 - fi - fi - - ensure-running: - desc: Ensure the development container is running - internal: true - silent: true - cmds: - - | - if ! docker ps | grep -q "{{.CONTAINER_NAME}}"; then - echo "Container {{.CONTAINER_NAME}} is not running. Starting it..." - task start - else - echo "Container {{.CONTAINER_NAME}} is already running." - fi - status: - - docker ps | grep -q "{{.CONTAINER_NAME}}" - - shell: - desc: Attach to development container shell - deps: [ensure-running] - silent: true - cmds: - - echo "Connecting to {{.CONTAINER_NAME}}..." - - docker exec -it {{.CONTAINER_NAME}} /bin/bash - - stop: - desc: Stop development container - silent: true - cmds: - - | - if docker ps | grep -q "{{.CONTAINER_NAME}}"; then - echo "Stopping {{.CONTAINER_NAME}} development container..." - docker stop {{.CONTAINER_NAME}} - else - echo "Container {{.CONTAINER_NAME}} is not running" - fi - - restart: - desc: Restart development container - silent: true - cmds: - - task: stop - - task: start diff --git a/applications/wg-easy/Dockerfile b/applications/wg-easy/Dockerfile deleted file mode 100644 index 950a9d66..00000000 --- a/applications/wg-easy/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -FROM --platform=$BUILDPLATFORM ubuntu:24.04 AS build - -ARG TARGETOS -ARG TARGETARCH - -# Tool versions -ARG REPLICATED_VERSION=0.101.1 -ARG TASK_VERSION=3.43.2 -ARG KUBECTL_VERSION=1.30.1 -ARG HELM_VERSION=3.17.3 -ARG HELMFILE_VERSION=0.171.0 -ARG YQ_VERSION=4.45.1 - -RUN apt-get update && apt-get install -y \ - curl \ - jq \ - yq \ - gpg \ - && rm -rf /var/lib/apt/lists/* - -# Install the Google Cloud tools following the instructions here: https://cloud.google.com/sdk/docs/install#deb -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ - && apt-get update \ - && apt-get install google-cloud-cli -y \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /tools - -# TODO: use checksums to verify the downloaded files - -# TODO: Use this to build for different platforms -RUN echo "=======================================================" -RUN echo "Building for TARGET PLATFORM: ${TARGETOS}_${TARGETARCH}" -RUN echo "=======================================================" - -# Download and extract the replicated binary -RUN curl -L -o replicated.tar.gz https://github.com/replicatedhq/replicated/releases/download/v${REPLICATED_VERSION}/replicated_${REPLICATED_VERSION}_linux_amd64.tar.gz \ - && tar -xzf replicated.tar.gz \ - && rm -f replicated.tar.gz LICENSE README.md - -# Download and extract the task binary -RUN curl -L -o task.tar.gz https://github.com/go-task/task/releases/download/v${TASK_VERSION}/task_${TARGETOS}_${TARGETARCH}.tar.gz \ - && tar -xzf task.tar.gz \ - && rm -rf task.tar.gz LICENSE README.md completion/ - -# Download and extract the kubectl binary -RUN curl -L -o kubectl https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${TARGETOS}/${TARGETARCH}/kubectl \ - && chmod +x kubectl - -# Download and extract the helm binary -RUN curl -L -o helm.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz \ - && tar -xzf helm.tar.gz \ - && mv ${TARGETOS}-${TARGETARCH}/helm . \ - && rm -rf ${TARGETOS}-${TARGETARCH} helm.tar.gz - -# Download helmfile binary -RUN curl -vL -o helmfile.tar.gz https://github.com/helmfile/helmfile/releases/download/v${HELMFILE_VERSION}/helmfile_${HELMFILE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz \ - && tar -xzf helmfile.tar.gz \ - && rm -f helmfile.tar.gz - -# Download and extract the yq binary -RUN curl -L -o yq https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_${TARGETOS}_${TARGETARCH} \ - && chmod +x yq - -WORKDIR /app - -ENV PATH="/tools:${PATH}" diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index 861775c6..5707be47 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -2,7 +2,7 @@ version: "3" includes: utils: ./taskfiles/utils.yml - dev: ./../taskfiles/docker.yml + dev: ./taskfiles/container.yml vars: # Application configuration @@ -31,7 +31,6 @@ vars: # Docker workflow configuration IMAGE_NAME: ttl.sh/wg-easy-dev CONTAINER_NAME: wg-easy-dev - IMAGE_TAG: latest tasks: default: diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile new file mode 100644 index 00000000..7c624514 --- /dev/null +++ b/applications/wg-easy/container/Containerfile @@ -0,0 +1,67 @@ +# Base image for all shared Dockerfiles for taskfiles +# Use this image as base image for app specific docker files +FROM --platform=$BUILDPLATFORM ubuntu:24.04 + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /tools + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive \ + HOME=/home/devuser + +# Install CLI tools +RUN apt-get update && apt-get install -y \ + curl \ + jq \ + yq \ + gnupg \ + sudo \ + + # Install Helm + && curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ + + # Install kubectl + && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x kubectl \ + && mv kubectl /usr/local/bin/ \ + + # Install Task + && sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin \ + + # Install Helmfile + && curl -Ls $(curl -s https://api.github.com/repos/helmfile/helmfile/releases/latest \ + | grep "browser_download_url.*linux_amd64.tar.gz" \ + | cut -d : -f 2,3 \ + | tr -d \") -o helmfile.tar.gz \ + && tar xf helmfile.tar.gz helmfile && rm helmfile.tar.gz \ + && mv helmfile /usr/local/bin/helmfile \ + + # Install Replicated CLI + && curl -Ls $(curl -s https://api.github.com/repos/replicatedhq/replicated/releases/latest \ + | grep "browser_download_url.*linux_amd64.tar.gz" \ + | cut -d : -f 2,3 \ + | tr -d \") -o replicated.tar.gz \ + && tar xf replicated.tar.gz replicated && rm replicated.tar.gz \ + && mv replicated /usr/local/bin/replicated \ + + # Install Google Cloud CLI + && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ + && apt-get update \ + && apt-get install google-cloud-cli -y \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user for better security +RUN groupadd -r devuser \ + && useradd -r -g devuser -m -s /bin/bash devuser \ + && echo "devuser ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/devuser + +# Set working directory +WORKDIR /app + +# Switch to non-root user +USER devuser + +CMD ["bash"] diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml new file mode 100644 index 00000000..7f4cd191 --- /dev/null +++ b/applications/wg-easy/taskfiles/container.yml @@ -0,0 +1,87 @@ +version: "3" + +# Development environment tasks +tasks: + build-image: + desc: Build development Docker image + vars: + IMAGE_NAME: '{{.IMAGE_NAME}}' + IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' + CONTAINERFILE: '{{.CONTAINERFILE | default "./container/Containerfile"}}' + BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' + requires: + vars: [IMAGE_NAME, IMAGE_TAG, CONTAINERFILE] + + cmds: + - docker build -t {{.IMAGE_NAME}} -f {{.CONTAINERFILE}} . + + start: + desc: Start development container in background + silent: true + cmds: + - task: start-implementation + + start-implementation: + desc: Start development container in background + silent: true + run: once + internal: true + vars: + IMAGE_NAME: '{{.IMAGE_NAME}}' + CONTAINER_NAME: '{{.CONTAINER_NAME}}' + IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' + requires: + vars: [IMAGE_NAME, CONTAINER_NAME] + + status: + - docker ps | grep -q "{{.CONTAINER_NAME}}" + cmds: + - | + # Start container with host networking for kubectl port-forward compatibility + CONTAINER_ID=$(docker run --rm --name {{.CONTAINER_NAME}} -d \ + -v $(pwd):/workspace \ + -e HOME=/home/devuser \ + -e USER=devuser \ + -w /workspace \ + {{.IMAGE_NAME}}:{{.IMAGE_TAG}} bash -c 'trap "exit" TERM; while :; do sleep 0.1; done') + + if [ $? -eq 0 ]; then + echo "Development container started successfully with ID: $CONTAINER_ID" + else + echo "Failed to start development container" + exit 1 + fi + + shell: + desc: Attach to development container shell + silent: true + requires: + vars: [CONTAINER_NAME] + deps: + - start-implementation + cmds: + - echo "Connecting to {{.CONTAINER_NAME}}..." + - docker exec -it {{.CONTAINER_NAME}} /bin/bash + + stop: + desc: Stop development container + silent: true + requires: + vars: [CONTAINER_NAME] + cmds: + - | + if docker ps | grep -q "{{.CONTAINER_NAME}}"; then + echo "Stopping {{.CONTAINER_NAME}} development container..." + docker stop {{.CONTAINER_NAME}} + else + echo "Container {{.CONTAINER_NAME}} is not running" + fi + + restart: + desc: Restart development container + silent: true + requires: + vars: [CONTAINER_NAME] + cmds: + - task: stop + - task: start From 3e5c1cd2242ba5e72a45ecac5c0e0c4dc57565ee Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 19:10:05 +0100 Subject: [PATCH 07/30] Update dev documentation Signed-off-by: Evans Mungai --- applications/wg-easy/docs/development-workflow.md | 13 ++++--------- applications/wg-easy/taskfiles/container.yml | 2 ++ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/applications/wg-easy/docs/development-workflow.md b/applications/wg-easy/docs/development-workflow.md index a457942b..fb82c036 100644 --- a/applications/wg-easy/docs/development-workflow.md +++ b/applications/wg-easy/docs/development-workflow.md @@ -18,14 +18,9 @@ The core philosophy of this workflow is to start simple and add complexity incre Before starting the development workflow, ensure you have the following tools installed: - **Task:** The task runner used in this project. ([Installation Guide](https://taskfile.dev/installation/)) -- **Replicated CLI:** For managing test clusters and application releases. ([Installation Guide](https://docs.replicated.com/reference/replicated-cli-installing)) -- **Helm:** The Kubernetes package manager. ([Installation Guide](https://helm.sh/docs/intro/install/)) -- **Helmfile:** For orchestrating Helm chart deployments. ([Installation Guide](https://github.com/helmfile/helmfile#installation)) -- **kubectl:** The Kubernetes command-line tool. ([Installation Guide](https://kubernetes.io/docs/tasks/tools/install-kubectl/)) -- **jq:** A command-line JSON processor. ([Download Page](https://stedolan.github.io/jq/download/)) -- **yq:** A command-line YAML processor. ([Installation Guide](https://github.com/mikefarah/yq#install)) -- **gcloud CLI:** Google Cloud command-line interface (optional, only required for GCP-specific tasks). ([Installation Guide](https://cloud.google.com/sdk/docs/install)) -- **Standard Unix Utilities:** `find`, `xargs`, `grep`, `awk`, `wc`, `tr`, `cp`, `mv`, `rm`, `mkdir`, `echo`, `sleep`, `test`, `eval` (typically available by default on Linux and macOS). +- **Docker:** Container runtime for local development. ([Installation Guide](https://docs.docker.com/get-docker/)) + +All other tools will be automatically provided through task commands and Docker containers. ## Workflow Stages @@ -177,7 +172,7 @@ Test multiple charts working together using Helmfile orchestration. # Check if issuers are correctly using cert-manager kubectl get clusterissuers kubectl get issuers -A - + # Verify Traefik routes kubectl get ingressroutes -A ``` diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 7f4cd191..7b70bbc3 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -21,6 +21,8 @@ tasks: cmds: - task: start-implementation + # Start development container in background. + # It's internal because it's used by shell and start tasks. start-implementation: desc: Start development container in background silent: true From 30b77819239d3686b839c52e1afdcf3d6d0b32ae Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 19:22:14 +0100 Subject: [PATCH 08/30] Copy replicated auth token to container Signed-off-by: Evans Mungai --- applications/wg-easy/taskfiles/container.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 7b70bbc3..b6c56741 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -33,7 +33,7 @@ tasks: CONTAINER_NAME: '{{.CONTAINER_NAME}}' IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' requires: - vars: [IMAGE_NAME, CONTAINER_NAME] + vars: [IMAGE_NAME, CONTAINER_NAME, REPLICATED_API_TOKEN] status: - docker ps | grep -q "{{.CONTAINER_NAME}}" @@ -44,6 +44,7 @@ tasks: -v $(pwd):/workspace \ -e HOME=/home/devuser \ -e USER=devuser \ + -e REPLICATED_API_TOKEN={{ .REPLICATED_API_TOKEN }} \ -w /workspace \ {{.IMAGE_NAME}}:{{.IMAGE_TAG}} bash -c 'trap "exit" TERM; while :; do sleep 0.1; done') From e26440ecb34e45ab382f91867d3fc8c170d4b42d Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 19:32:25 +0100 Subject: [PATCH 09/30] Update dev documentation Signed-off-by: Evans Mungai --- applications/wg-easy/docs/development-workflow.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/applications/wg-easy/docs/development-workflow.md b/applications/wg-easy/docs/development-workflow.md index fb82c036..77da9a82 100644 --- a/applications/wg-easy/docs/development-workflow.md +++ b/applications/wg-easy/docs/development-workflow.md @@ -92,6 +92,17 @@ Configure chart values and create or modify templates. ### Stage 3: Local Validation with helm template +> [!IMPORTANT] +> Tools required by tasks in this project will be made available in a container. Run the command below to start the dev environment + +``` +# Build tools image. Run this command once. +task dev:build-image + +# Open shell to execute tasks +task dev:shell +``` + Validate chart templates locally without deploying to a cluster. 1. Run helm template to render the chart and inspect manifests: From 0e669a9b76312e1612e2c6b49a44421524d535a8 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 19:35:00 +0100 Subject: [PATCH 10/30] Fix grammer Signed-off-by: Evans Mungai --- applications/wg-easy/docs/development-workflow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/wg-easy/docs/development-workflow.md b/applications/wg-easy/docs/development-workflow.md index 77da9a82..e6906c48 100644 --- a/applications/wg-easy/docs/development-workflow.md +++ b/applications/wg-easy/docs/development-workflow.md @@ -93,7 +93,7 @@ Configure chart values and create or modify templates. ### Stage 3: Local Validation with helm template > [!IMPORTANT] -> Tools required by tasks in this project will be made available in a container. Run the command below to start the dev environment +> Tools required by tasks in this project will be made available in a container. Run the commands below to start the dev environment ``` # Build tools image. Run this command once. From 0da51beda86c53a09792dd8d18821ce640541c29 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Fri, 25 Apr 2025 19:41:16 +0100 Subject: [PATCH 11/30] Add SHELL env required by replicated shell subcommand Signed-off-by: Evans Mungai --- applications/wg-easy/container/Containerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 7c624514..82274492 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -9,7 +9,8 @@ WORKDIR /tools # Set environment variables ENV DEBIAN_FRONTEND=noninteractive \ - HOME=/home/devuser + HOME=/home/devuser \ + SHELL=/bin/bash # Install CLI tools RUN apt-get update && apt-get install -y \ From a75abac07327dc62d18184826baf06404e469826 Mon Sep 17 00:00:00 2001 From: Scott Rigby Date: Mon, 28 Apr 2025 06:52:26 -0400 Subject: [PATCH 12/30] Default to Podman, allow Docker optionally (#54) Signed-off-by: Scott Rigby --- applications/wg-easy/Taskfile.yaml | 3 ++- applications/wg-easy/container/Containerfile | 4 ++-- applications/wg-easy/docs/development-workflow.md | 4 ++-- applications/wg-easy/taskfiles/container.yml | 14 +++++++------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index 5707be47..e7e6d1e2 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -28,9 +28,10 @@ vars: GCP_ZONE: '{{.GCP_ZONE | default "us-central1-a"}}' VM_NAME: '{{.VM_NAME | default (printf "%s-dev" (or (env "GUSER") "user"))}}' - # Docker workflow configuration + # Container workflow configuration IMAGE_NAME: ttl.sh/wg-easy-dev CONTAINER_NAME: wg-easy-dev + CONTAINER_RUNTIME: '{{.CONTAINER_RUNTIME | default "podman"}}' tasks: default: diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 82274492..0044c779 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -1,5 +1,5 @@ -# Base image for all shared Dockerfiles for taskfiles -# Use this image as base image for app specific docker files +# Base image for all shared Containerfiles for taskfiles +# Use this image as base image for app specific container files FROM --platform=$BUILDPLATFORM ubuntu:24.04 ARG TARGETOS diff --git a/applications/wg-easy/docs/development-workflow.md b/applications/wg-easy/docs/development-workflow.md index e6906c48..13f5ce3a 100644 --- a/applications/wg-easy/docs/development-workflow.md +++ b/applications/wg-easy/docs/development-workflow.md @@ -18,9 +18,9 @@ The core philosophy of this workflow is to start simple and add complexity incre Before starting the development workflow, ensure you have the following tools installed: - **Task:** The task runner used in this project. ([Installation Guide](https://taskfile.dev/installation/)) -- **Docker:** Container runtime for local development. ([Installation Guide](https://docs.docker.com/get-docker/)) +- **Container runtime tool** Either [Podman](https://podman.io/docs/installation) (default) or [Docker](https://docs.docker.com/get-docker/) for local development -All other tools will be automatically provided through task commands and Docker containers. +All other tools will be automatically provided through task commands and containers. ## Workflow Stages diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index b6c56741..badcd03a 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -3,7 +3,7 @@ version: "3" # Development environment tasks tasks: build-image: - desc: Build development Docker image + desc: Build development container image vars: IMAGE_NAME: '{{.IMAGE_NAME}}' IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' @@ -13,7 +13,7 @@ tasks: vars: [IMAGE_NAME, IMAGE_TAG, CONTAINERFILE] cmds: - - docker build -t {{.IMAGE_NAME}} -f {{.CONTAINERFILE}} . + - '{{.CONTAINER_RUNTIME}} build -t {{.IMAGE_NAME}} -f {{.CONTAINERFILE}} .' start: desc: Start development container in background @@ -36,11 +36,11 @@ tasks: vars: [IMAGE_NAME, CONTAINER_NAME, REPLICATED_API_TOKEN] status: - - docker ps | grep -q "{{.CONTAINER_NAME}}" + - '{{.CONTAINER_RUNTIME}} ps | grep -q "{{.CONTAINER_NAME}}"' cmds: - | # Start container with host networking for kubectl port-forward compatibility - CONTAINER_ID=$(docker run --rm --name {{.CONTAINER_NAME}} -d \ + CONTAINER_ID=$({{.CONTAINER_RUNTIME}} run --rm --name {{.CONTAINER_NAME}} -d \ -v $(pwd):/workspace \ -e HOME=/home/devuser \ -e USER=devuser \ @@ -64,7 +64,7 @@ tasks: - start-implementation cmds: - echo "Connecting to {{.CONTAINER_NAME}}..." - - docker exec -it {{.CONTAINER_NAME}} /bin/bash + - '{{.CONTAINER_RUNTIME}} exec -it {{.CONTAINER_NAME}} /bin/bash' stop: desc: Stop development container @@ -73,9 +73,9 @@ tasks: vars: [CONTAINER_NAME] cmds: - | - if docker ps | grep -q "{{.CONTAINER_NAME}}"; then + if {{.CONTAINER_RUNTIME}} ps | grep -q "{{.CONTAINER_NAME}}"; then echo "Stopping {{.CONTAINER_NAME}} development container..." - docker stop {{.CONTAINER_NAME}} + {{.CONTAINER_RUNTIME}} stop {{.CONTAINER_NAME}} else echo "Container {{.CONTAINER_NAME}} is not running" fi From e77f4f09e539bb0c0e0267e5dc01b40613ceb260 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Mon, 28 Apr 2025 12:10:46 +0100 Subject: [PATCH 13/30] Prepend DEV to dev container variables Signed-off-by: Evans Mungai --- applications/wg-easy/Taskfile.yaml | 6 ++- applications/wg-easy/taskfiles/container.yml | 40 ++++++++++---------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index e7e6d1e2..400e8e0f 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -29,8 +29,10 @@ vars: VM_NAME: '{{.VM_NAME | default (printf "%s-dev" (or (env "GUSER") "user"))}}' # Container workflow configuration - IMAGE_NAME: ttl.sh/wg-easy-dev - CONTAINER_NAME: wg-easy-dev + DEV_CONTAINER_REPOSITORY: '{{.DEV_CONTAINER_REPOSITORY | default "docker.io/wg-easy"}}' + DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE | default "wg-easy"}}' + DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' + DEV_CONTAINER_NAME: '{{.DEV_CONTAINER_NAME | default "wg-easy-dev"}}' CONTAINER_RUNTIME: '{{.CONTAINER_RUNTIME | default "podman"}}' tasks: diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index badcd03a..7ea8ee83 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -5,15 +5,14 @@ tasks: build-image: desc: Build development container image vars: - IMAGE_NAME: '{{.IMAGE_NAME}}' - IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' + DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' CONTAINERFILE: '{{.CONTAINERFILE | default "./container/Containerfile"}}' BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' requires: - vars: [IMAGE_NAME, IMAGE_TAG, CONTAINERFILE] + vars: [DEV_CONTAINER_REPOSITORY, DEV_CONTAINER_IMAGE, CONTAINERFILE] cmds: - - '{{.CONTAINER_RUNTIME}} build -t {{.IMAGE_NAME}} -f {{.CONTAINERFILE}} .' + - '{{.CONTAINER_RUNTIME}} build -t {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} -f {{.CONTAINERFILE}} .' start: desc: Start development container in background @@ -29,24 +28,25 @@ tasks: run: once internal: true vars: - IMAGE_NAME: '{{.IMAGE_NAME}}' - CONTAINER_NAME: '{{.CONTAINER_NAME}}' - IMAGE_TAG: '{{.IMAGE_TAG | default "latest"}}' + DEV_CONTAINER_REPOSITORY: '{{.DEV_CONTAINER_REPOSITORY}}' + DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE}}' + DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' + DEV_CONTAINER_NAME: '{{.DEV_CONTAINER_NAME}}' requires: - vars: [IMAGE_NAME, CONTAINER_NAME, REPLICATED_API_TOKEN] + vars: [DEV_CONTAINER_REPOSITORY, DEV_CONTAINER_IMAGE, DEV_CONTAINER_TAG, DEV_CONTAINER_NAME, REPLICATED_API_TOKEN] status: - - '{{.CONTAINER_RUNTIME}} ps | grep -q "{{.CONTAINER_NAME}}"' + - '{{.CONTAINER_RUNTIME}} ps | grep -q "{{.DEV_CONTAINER_NAME}}"' cmds: - | # Start container with host networking for kubectl port-forward compatibility - CONTAINER_ID=$({{.CONTAINER_RUNTIME}} run --rm --name {{.CONTAINER_NAME}} -d \ + CONTAINER_ID=$({{.CONTAINER_RUNTIME}} run --rm --name {{.DEV_CONTAINER_NAME}} -d \ -v $(pwd):/workspace \ -e HOME=/home/devuser \ -e USER=devuser \ -e REPLICATED_API_TOKEN={{ .REPLICATED_API_TOKEN }} \ -w /workspace \ - {{.IMAGE_NAME}}:{{.IMAGE_TAG}} bash -c 'trap "exit" TERM; while :; do sleep 0.1; done') + {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} bash -c 'trap "exit" TERM; while :; do sleep 0.1; done') if [ $? -eq 0 ]; then echo "Development container started successfully with ID: $CONTAINER_ID" @@ -59,32 +59,32 @@ tasks: desc: Attach to development container shell silent: true requires: - vars: [CONTAINER_NAME] + vars: [DEV_CONTAINER_NAME] deps: - start-implementation cmds: - - echo "Connecting to {{.CONTAINER_NAME}}..." - - '{{.CONTAINER_RUNTIME}} exec -it {{.CONTAINER_NAME}} /bin/bash' + - echo "Connecting to {{.DEV_CONTAINER_NAME}}..." + - '{{.CONTAINER_RUNTIME}} exec -it {{.DEV_CONTAINER_NAME}} /bin/bash' stop: desc: Stop development container silent: true requires: - vars: [CONTAINER_NAME] + vars: [DEV_CONTAINER_NAME] cmds: - | - if {{.CONTAINER_RUNTIME}} ps | grep -q "{{.CONTAINER_NAME}}"; then - echo "Stopping {{.CONTAINER_NAME}} development container..." - {{.CONTAINER_RUNTIME}} stop {{.CONTAINER_NAME}} + if {{.CONTAINER_RUNTIME}} ps | grep -q "{{.DEV_CONTAINER_NAME}}"; then + echo "Stopping {{.DEV_CONTAINER_NAME}} development container..." + {{.CONTAINER_RUNTIME}} stop {{.DEV_CONTAINER_NAME}} else - echo "Container {{.CONTAINER_NAME}} is not running" + echo "Container {{.DEV_CONTAINER_NAME}} is not running" fi restart: desc: Restart development container silent: true requires: - vars: [CONTAINER_NAME] + vars: [DEV_CONTAINER_NAME] cmds: - task: stop - task: start From 68ecc8dfa87ffc4ec9521e38c0d4f91826b67c84 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Mon, 28 Apr 2025 13:00:00 +0100 Subject: [PATCH 14/30] Changes as per review comments Signed-off-by: Evans Mungai --- applications/wg-easy/container/Containerfile | 12 +++++++----- applications/wg-easy/taskfiles/container.yml | 5 +---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 0044c779..e809594d 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -18,7 +18,6 @@ RUN apt-get update && apt-get install -y \ jq \ yq \ gnupg \ - sudo \ # Install Helm && curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ @@ -52,15 +51,18 @@ RUN apt-get update && apt-get install -y \ && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ && apt-get update \ && apt-get install google-cloud-cli -y \ + + # Clean up + && apt-get purge -y \ + curl \ + gnupg \ && rm -rf /var/lib/apt/lists/* # Create a non-root user for better security -RUN groupadd -r devuser \ - && useradd -r -g devuser -m -s /bin/bash devuser \ - && echo "devuser ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/devuser +RUN groupadd -r devuser && useradd -r -g devuser -m -s /bin/bash devuser # Set working directory -WORKDIR /app +WORKDIR /workspace # Switch to non-root user USER devuser diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 7ea8ee83..298fd2d3 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -42,11 +42,8 @@ tasks: # Start container with host networking for kubectl port-forward compatibility CONTAINER_ID=$({{.CONTAINER_RUNTIME}} run --rm --name {{.DEV_CONTAINER_NAME}} -d \ -v $(pwd):/workspace \ - -e HOME=/home/devuser \ - -e USER=devuser \ -e REPLICATED_API_TOKEN={{ .REPLICATED_API_TOKEN }} \ - -w /workspace \ - {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} bash -c 'trap "exit" TERM; while :; do sleep 0.1; done') + {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} bash -c 'trap "exit 0" TERM; sleep infinity & wait') if [ $? -eq 0 ]; then echo "Development container started successfully with ID: $CONTAINER_ID" From cdcc99148475ffa4e67fe93e4835716eb73523da Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Mon, 28 Apr 2025 16:41:22 +0100 Subject: [PATCH 15/30] Add shell completions and some colour to terminal Signed-off-by: Evans Mungai --- applications/wg-easy/container/Containerfile | 11 ++++++++- applications/wg-easy/container/entrypoint.sh | 10 ++++++++ .../wg-easy/container/tool-completions.sh | 24 +++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 applications/wg-easy/container/entrypoint.sh create mode 100644 applications/wg-easy/container/tool-completions.sh diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index e809594d..7e354df1 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y \ jq \ yq \ gnupg \ + bash-completion \ # Install Helm && curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ @@ -61,10 +62,18 @@ RUN apt-get update && apt-get install -y \ # Create a non-root user for better security RUN groupadd -r devuser && useradd -r -g devuser -m -s /bin/bash devuser +# Copy shell completion scripts +COPY container/tool-completions.sh tool-completions.sh + +# Copy entrypoint script +COPY container/entrypoint.sh entrypoint.sh +RUN chmod +x entrypoint.sh + # Set working directory WORKDIR /workspace # Switch to non-root user USER devuser -CMD ["bash"] +# Set entrypoint +ENTRYPOINT ["/tools/entrypoint.sh", "-l"] diff --git a/applications/wg-easy/container/entrypoint.sh b/applications/wg-easy/container/entrypoint.sh new file mode 100644 index 00000000..7f4e98c3 --- /dev/null +++ b/applications/wg-easy/container/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Uncomment force_color_prompt in bashrc +sed -i 's/#force_color_prompt=yes/force_color_prompt=yes/' $HOME/.bashrc + +# Source the tool completions +echo "source /tools/tool-completions.sh" >> $HOME/.bashrc + +# Execute the passed command or default to bash +exec "$@" diff --git a/applications/wg-easy/container/tool-completions.sh b/applications/wg-easy/container/tool-completions.sh new file mode 100644 index 00000000..e872748b --- /dev/null +++ b/applications/wg-easy/container/tool-completions.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Enable bash completion +source /etc/bash_completion + +# kubectl completion +source <(kubectl completion bash) +alias k=kubectl +complete -o default -F __start_kubectl k + +# helm completion +source <(helm completion bash) + +# task completion +source <(task --completion bash) + +# helmfile completion +source <(helmfile completion bash) + +# replicated completion +source <(replicated completion bash) + +# gcloud completion +source /usr/share/google-cloud-sdk/completion.bash.inc From a05d485f6ea50718b1bc98330e370229f729f7b0 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Mon, 28 Apr 2025 16:44:15 +0100 Subject: [PATCH 16/30] Remove duplicate completion Signed-off-by: Evans Mungai --- applications/wg-easy/container/tool-completions.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/applications/wg-easy/container/tool-completions.sh b/applications/wg-easy/container/tool-completions.sh index e872748b..50edb3a1 100644 --- a/applications/wg-easy/container/tool-completions.sh +++ b/applications/wg-easy/container/tool-completions.sh @@ -1,8 +1,5 @@ #!/bin/bash -# Enable bash completion -source /etc/bash_completion - # kubectl completion source <(kubectl completion bash) alias k=kubectl From c8566f273d46c3ef684da9ddff5dbfaf06b282c3 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 10:36:06 +0100 Subject: [PATCH 17/30] Update applications/wg-easy/container/Containerfile Co-authored-by: Scott Rigby --- applications/wg-easy/container/Containerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 7e354df1..2f438afc 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -16,6 +16,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get update && apt-get install -y \ curl \ jq \ + less \ yq \ gnupg \ bash-completion \ From b8238724a909d5efac387bb1e2c1d004f43ac041 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 10:39:20 +0100 Subject: [PATCH 18/30] Update applications/wg-easy/docs/development-workflow.md Co-authored-by: Scott Rigby --- applications/wg-easy/docs/development-workflow.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/applications/wg-easy/docs/development-workflow.md b/applications/wg-easy/docs/development-workflow.md index 13f5ce3a..8516e95d 100644 --- a/applications/wg-easy/docs/development-workflow.md +++ b/applications/wg-easy/docs/development-workflow.md @@ -99,6 +99,9 @@ Configure chart values and create or modify templates. # Build tools image. Run this command once. task dev:build-image +# Start/restart tools container. Idempotent. +task dev:restart + # Open shell to execute tasks task dev:shell ``` From 4e7619edd0cf3a63d84107358c5b8a887f194591 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 12:59:31 +0100 Subject: [PATCH 19/30] Remove unnecessary shell-implementation task Signed-off-by: Evans Mungai --- applications/wg-easy/taskfiles/container.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 298fd2d3..07241111 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -14,24 +14,14 @@ tasks: cmds: - '{{.CONTAINER_RUNTIME}} build -t {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} -f {{.CONTAINERFILE}} .' - start: - desc: Start development container in background - silent: true - cmds: - - task: start-implementation - # Start development container in background. # It's internal because it's used by shell and start tasks. - start-implementation: + start: desc: Start development container in background silent: true run: once - internal: true vars: - DEV_CONTAINER_REPOSITORY: '{{.DEV_CONTAINER_REPOSITORY}}' - DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE}}' DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' - DEV_CONTAINER_NAME: '{{.DEV_CONTAINER_NAME}}' requires: vars: [DEV_CONTAINER_REPOSITORY, DEV_CONTAINER_IMAGE, DEV_CONTAINER_TAG, DEV_CONTAINER_NAME, REPLICATED_API_TOKEN] @@ -58,7 +48,7 @@ tasks: requires: vars: [DEV_CONTAINER_NAME] deps: - - start-implementation + - task: start cmds: - echo "Connecting to {{.DEV_CONTAINER_NAME}}..." - '{{.CONTAINER_RUNTIME}} exec -it {{.DEV_CONTAINER_NAME}} /bin/bash' From 6649ed731f2ad2e9848fd6431f7fd0c3d3bbe4e6 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 13:09:49 +0100 Subject: [PATCH 20/30] Improvements in Containerfile Signed-off-by: Evans Mungai --- applications/wg-easy/container/Containerfile | 30 +++++++++----------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 2f438afc..025cec5b 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ HOME=/home/devuser \ SHELL=/bin/bash -# Install CLI tools +# Install debian packages RUN apt-get update && apt-get install -y \ curl \ jq \ @@ -21,8 +21,18 @@ RUN apt-get update && apt-get install -y \ gnupg \ bash-completion \ - # Install Helm - && curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ + # Install Google Cloud CLI + && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ + && apt-get update \ + && apt-get install google-cloud-cli -y \ + + # Clean up + && apt-get purge -y gnupg \ + && rm -rf /var/lib/apt/lists/* + +# Install helm +RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ # Install kubectl && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ @@ -46,19 +56,7 @@ RUN apt-get update && apt-get install -y \ | cut -d : -f 2,3 \ | tr -d \") -o replicated.tar.gz \ && tar xf replicated.tar.gz replicated && rm replicated.tar.gz \ - && mv replicated /usr/local/bin/replicated \ - - # Install Google Cloud CLI - && echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \ - && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg \ - && apt-get update \ - && apt-get install google-cloud-cli -y \ - - # Clean up - && apt-get purge -y \ - curl \ - gnupg \ - && rm -rf /var/lib/apt/lists/* + && mv replicated /usr/local/bin/replicated # Create a non-root user for better security RUN groupadd -r devuser && useradd -r -g devuser -m -s /bin/bash devuser From cba7c04830ce8c9d48a1248e36ed14901c85aa85 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 13:15:25 +0100 Subject: [PATCH 21/30] Remove unnecessary comment Signed-off-by: Evans Mungai --- applications/wg-easy/taskfiles/container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 07241111..99789804 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -15,7 +15,6 @@ tasks: - '{{.CONTAINER_RUNTIME}} build -t {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} -f {{.CONTAINERFILE}} .' # Start development container in background. - # It's internal because it's used by shell and start tasks. start: desc: Start development container in background silent: true From 5b4b5aa5683ef57b167c98e6d0051870faddb5de Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 14:33:31 +0100 Subject: [PATCH 22/30] Build and publish wg-easy tools image to ghcr Signed-off-by: Evans Mungai --- .github/workflows/wg-easy-image.yml | 60 ++++++++++++++++++++ applications/wg-easy/Taskfile.yaml | 6 +- applications/wg-easy/taskfiles/container.yml | 8 +-- 3 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/wg-easy-image.yml diff --git a/.github/workflows/wg-easy-image.yml b/.github/workflows/wg-easy-image.yml new file mode 100644 index 00000000..8a1c833e --- /dev/null +++ b/.github/workflows/wg-easy-image.yml @@ -0,0 +1,60 @@ +name: WG-Easy Image CI + +on: + push: + branches: [ main ] + paths: + - 'applications/wg-easy/container/**' + - '.github/workflows/wg-easy-container.yml' + pull_request: + paths: + - 'applications/wg-easy/container/**' + - '.github/workflows/wg-easy-container.yml' + workflow_dispatch: + +env: + DEV_CONTAINER_REGISTRY: ghcr.io + DEV_CONTAINER_IMAGE: platform-examples/wg-easy-tools + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.DEV_CONTAINER_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DEV_CONTAINER_REGISTRY }}/${{ env.DEV_CONTAINER_IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,format=short + type=ref,event=branch + type=ref,event=pr + + - name: Build and push image + uses: docker/build-push-action@v6 + with: + context: applications/wg-easy + file: applications/wg-easy/container/Containerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index 400e8e0f..276308f1 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -29,10 +29,10 @@ vars: VM_NAME: '{{.VM_NAME | default (printf "%s-dev" (or (env "GUSER") "user"))}}' # Container workflow configuration - DEV_CONTAINER_REPOSITORY: '{{.DEV_CONTAINER_REPOSITORY | default "docker.io/wg-easy"}}' - DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE | default "wg-easy"}}' + DEV_CONTAINER_REGISTRY: '{{.DEV_CONTAINER_REGISTRY | default "ghcr.io"}}' + DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE | default "platform-examples/wg-easy-tools"}}' DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' - DEV_CONTAINER_NAME: '{{.DEV_CONTAINER_NAME | default "wg-easy-dev"}}' + DEV_CONTAINER_NAME: '{{.DEV_CONTAINER_NAME | default "wg-easy-tools"}}' CONTAINER_RUNTIME: '{{.CONTAINER_RUNTIME | default "podman"}}' tasks: diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 99789804..25dbad82 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -9,10 +9,10 @@ tasks: CONTAINERFILE: '{{.CONTAINERFILE | default "./container/Containerfile"}}' BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' requires: - vars: [DEV_CONTAINER_REPOSITORY, DEV_CONTAINER_IMAGE, CONTAINERFILE] + vars: [DEV_CONTAINER_REGISTRY, DEV_CONTAINER_IMAGE, CONTAINERFILE] cmds: - - '{{.CONTAINER_RUNTIME}} build -t {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} -f {{.CONTAINERFILE}} .' + - '{{.CONTAINER_RUNTIME}} build -t {{.DEV_CONTAINER_REGISTRY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} -f {{.CONTAINERFILE}} .' # Start development container in background. start: @@ -22,7 +22,7 @@ tasks: vars: DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' requires: - vars: [DEV_CONTAINER_REPOSITORY, DEV_CONTAINER_IMAGE, DEV_CONTAINER_TAG, DEV_CONTAINER_NAME, REPLICATED_API_TOKEN] + vars: [DEV_CONTAINER_REGISTRY, DEV_CONTAINER_IMAGE, DEV_CONTAINER_TAG, DEV_CONTAINER_NAME, REPLICATED_API_TOKEN] status: - '{{.CONTAINER_RUNTIME}} ps | grep -q "{{.DEV_CONTAINER_NAME}}"' @@ -32,7 +32,7 @@ tasks: CONTAINER_ID=$({{.CONTAINER_RUNTIME}} run --rm --name {{.DEV_CONTAINER_NAME}} -d \ -v $(pwd):/workspace \ -e REPLICATED_API_TOKEN={{ .REPLICATED_API_TOKEN }} \ - {{.DEV_CONTAINER_REPOSITORY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} bash -c 'trap "exit 0" TERM; sleep infinity & wait') + {{.DEV_CONTAINER_REGISTRY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} bash -c 'trap "exit 0" TERM; sleep infinity & wait') if [ $? -eq 0 ]; then echo "Development container started successfully with ID: $CONTAINER_ID" From 3c23789ba217dfae0d46e4c4b774091e82f7d8bc Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 14:37:26 +0100 Subject: [PATCH 23/30] Update paths to check in workflow file Signed-off-by: Evans Mungai --- .github/workflows/wg-easy-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wg-easy-image.yml b/.github/workflows/wg-easy-image.yml index 8a1c833e..ceec8389 100644 --- a/.github/workflows/wg-easy-image.yml +++ b/.github/workflows/wg-easy-image.yml @@ -4,12 +4,12 @@ on: push: branches: [ main ] paths: - - 'applications/wg-easy/container/**' - - '.github/workflows/wg-easy-container.yml' + - 'applications/wg-easy/**' + - '.github/workflows/wg-easy-image.yml' pull_request: paths: - - 'applications/wg-easy/container/**' - - '.github/workflows/wg-easy-container.yml' + - 'applications/wg-easy/**' + - '.github/workflows/wg-easy-image.yml' workflow_dispatch: env: From 1226492f85127603c59ff0c8627f44e8e01f644c Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 14:44:49 +0100 Subject: [PATCH 24/30] Fix repo name Signed-off-by: Evans Mungai --- .github/workflows/wg-easy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wg-easy-image.yml b/.github/workflows/wg-easy-image.yml index ceec8389..c47d7ca4 100644 --- a/.github/workflows/wg-easy-image.yml +++ b/.github/workflows/wg-easy-image.yml @@ -14,7 +14,7 @@ on: env: DEV_CONTAINER_REGISTRY: ghcr.io - DEV_CONTAINER_IMAGE: platform-examples/wg-easy-tools + DEV_CONTAINER_IMAGE: replicatedhq/platform-examples/wg-easy-tools jobs: build-and-push: From 629cb1edc6e7bfd90e213b119cff1385563a4784 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 14:45:04 +0100 Subject: [PATCH 25/30] Push in PR Signed-off-by: Evans Mungai --- .github/workflows/wg-easy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wg-easy-image.yml b/.github/workflows/wg-easy-image.yml index c47d7ca4..67d56581 100644 --- a/.github/workflows/wg-easy-image.yml +++ b/.github/workflows/wg-easy-image.yml @@ -53,7 +53,7 @@ jobs: with: context: applications/wg-easy file: applications/wg-easy/container/Containerfile - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha From 0825b3e34312bf2c14c9c04ac959775e73f9fdc7 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 14:54:34 +0100 Subject: [PATCH 26/30] Publish multiarch images Signed-off-by: Evans Mungai --- .github/workflows/wg-easy-image.yml | 4 ++++ applications/wg-easy/container/Containerfile | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wg-easy-image.yml b/.github/workflows/wg-easy-image.yml index 67d56581..e5d4a29a 100644 --- a/.github/workflows/wg-easy-image.yml +++ b/.github/workflows/wg-easy-image.yml @@ -27,6 +27,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -53,6 +56,7 @@ jobs: with: context: applications/wg-easy file: applications/wg-easy/container/Containerfile + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 025cec5b..6419f7aa 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -1,9 +1,6 @@ # Base image for all shared Containerfiles for taskfiles # Use this image as base image for app specific container files -FROM --platform=$BUILDPLATFORM ubuntu:24.04 - -ARG TARGETOS -ARG TARGETARCH +FROM ubuntu:24.04 WORKDIR /tools From 8889f4722c469591c7abefe403464b1f4863af8e Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 15:12:09 +0100 Subject: [PATCH 27/30] Download architecture specific binaries Signed-off-by: Evans Mungai --- applications/wg-easy/Taskfile.yaml | 2 +- applications/wg-easy/container/Containerfile | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/applications/wg-easy/Taskfile.yaml b/applications/wg-easy/Taskfile.yaml index 276308f1..d35852a3 100644 --- a/applications/wg-easy/Taskfile.yaml +++ b/applications/wg-easy/Taskfile.yaml @@ -30,7 +30,7 @@ vars: # Container workflow configuration DEV_CONTAINER_REGISTRY: '{{.DEV_CONTAINER_REGISTRY | default "ghcr.io"}}' - DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE | default "platform-examples/wg-easy-tools"}}' + DEV_CONTAINER_IMAGE: '{{.DEV_CONTAINER_IMAGE | default "replicatedhq/platform-examples/wg-easy-tools"}}' DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' DEV_CONTAINER_NAME: '{{.DEV_CONTAINER_NAME | default "wg-easy-tools"}}' CONTAINER_RUNTIME: '{{.CONTAINER_RUNTIME | default "podman"}}' diff --git a/applications/wg-easy/container/Containerfile b/applications/wg-easy/container/Containerfile index 6419f7aa..ccde16dc 100644 --- a/applications/wg-easy/container/Containerfile +++ b/applications/wg-easy/container/Containerfile @@ -1,6 +1,9 @@ # Base image for all shared Containerfiles for taskfiles # Use this image as base image for app specific container files -FROM ubuntu:24.04 +FROM --platform=$BUILDPLATFORM ubuntu:24.04 + +ARG TARGETOS +ARG TARGETARCH WORKDIR /tools @@ -32,7 +35,7 @@ RUN apt-get update && apt-get install -y \ RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash \ # Install kubectl - && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" \ && chmod +x kubectl \ && mv kubectl /usr/local/bin/ \ @@ -41,7 +44,7 @@ RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | b # Install Helmfile && curl -Ls $(curl -s https://api.github.com/repos/helmfile/helmfile/releases/latest \ - | grep "browser_download_url.*linux_amd64.tar.gz" \ + | grep "browser_download_url.*linux_${TARGETARCH}.tar.gz" \ | cut -d : -f 2,3 \ | tr -d \") -o helmfile.tar.gz \ && tar xf helmfile.tar.gz helmfile && rm helmfile.tar.gz \ From 83c1dc040584535f440f0f9db27a8d34b1fbdb61 Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Tue, 29 Apr 2025 15:21:08 +0100 Subject: [PATCH 28/30] Only push when in main branch Signed-off-by: Evans Mungai --- .github/workflows/wg-easy-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wg-easy-image.yml b/.github/workflows/wg-easy-image.yml index e5d4a29a..c22ca3b3 100644 --- a/.github/workflows/wg-easy-image.yml +++ b/.github/workflows/wg-easy-image.yml @@ -57,7 +57,7 @@ jobs: context: applications/wg-easy file: applications/wg-easy/container/Containerfile platforms: linux/amd64,linux/arm64 - push: true + push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha From 4de1ef6fc0a92022d25ddd39f9563ccd65b16d5a Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Wed, 30 Apr 2025 15:09:21 +0100 Subject: [PATCH 29/30] Check if image exists locally before pulling or building When starting the development container, check if the image exists locally before pulling or building it. Signed-off-by: Evans Mungai --- applications/wg-easy/taskfiles/container.yml | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/applications/wg-easy/taskfiles/container.yml b/applications/wg-easy/taskfiles/container.yml index 25dbad82..77e94ca8 100644 --- a/applications/wg-easy/taskfiles/container.yml +++ b/applications/wg-easy/taskfiles/container.yml @@ -5,14 +5,14 @@ tasks: build-image: desc: Build development container image vars: - DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' + IMAGE_NAME: '{{.DEV_CONTAINER_REGISTRY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}}' CONTAINERFILE: '{{.CONTAINERFILE | default "./container/Containerfile"}}' BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' requires: vars: [DEV_CONTAINER_REGISTRY, DEV_CONTAINER_IMAGE, CONTAINERFILE] cmds: - - '{{.CONTAINER_RUNTIME}} build -t {{.DEV_CONTAINER_REGISTRY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} -f {{.CONTAINERFILE}} .' + - '{{.CONTAINER_RUNTIME}} build -t {{.IMAGE_NAME}} -f {{.CONTAINERFILE}} .' # Start development container in background. start: @@ -20,7 +20,9 @@ tasks: silent: true run: once vars: - DEV_CONTAINER_TAG: '{{.DEV_CONTAINER_TAG | default "latest"}}' + IMAGE_NAME: '{{.DEV_CONTAINER_REGISTRY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}}' + CONTAINERFILE: '{{.CONTAINERFILE | default "./container/Containerfile"}}' + BUILD_ARGS: '{{.BUILD_ARGS | default ""}}' requires: vars: [DEV_CONTAINER_REGISTRY, DEV_CONTAINER_IMAGE, DEV_CONTAINER_TAG, DEV_CONTAINER_NAME, REPLICATED_API_TOKEN] @@ -28,11 +30,23 @@ tasks: - '{{.CONTAINER_RUNTIME}} ps | grep -q "{{.DEV_CONTAINER_NAME}}"' cmds: - | + # Check if the image exists locally + # If not, pull it from the registry + # If that fails, build it locally + if ! {{.CONTAINER_RUNTIME}} image exists {{.IMAGE_NAME}}; then + echo "Image {{.IMAGE_NAME}} not found locally." + echo "Attempting to pull the image..." + if ! {{.CONTAINER_RUNTIME}} pull {{.IMAGE_NAME}}; then + echo "Failed to pull image. Building it locally..." + {{.CONTAINER_RUNTIME}} build -t {{.IMAGE_NAME}} -f {{.CONTAINERFILE}} . + fi + fi + # Start container with host networking for kubectl port-forward compatibility CONTAINER_ID=$({{.CONTAINER_RUNTIME}} run --rm --name {{.DEV_CONTAINER_NAME}} -d \ -v $(pwd):/workspace \ -e REPLICATED_API_TOKEN={{ .REPLICATED_API_TOKEN }} \ - {{.DEV_CONTAINER_REGISTRY}}/{{.DEV_CONTAINER_IMAGE}}:{{.DEV_CONTAINER_TAG}} bash -c 'trap "exit 0" TERM; sleep infinity & wait') + {{.IMAGE_NAME}} bash -c 'trap "exit 0" TERM; sleep infinity & wait') if [ $? -eq 0 ]; then echo "Development container started successfully with ID: $CONTAINER_ID" From a3ef59ef347db981c26df7bd1688fc6b4f86d6cb Mon Sep 17 00:00:00 2001 From: Evans Mungai Date: Wed, 30 Apr 2025 15:14:36 +0100 Subject: [PATCH 30/30] Remove step to build image in docs dev:shell automatically builds the image if it doesn't exist locally Signed-off-by: Evans Mungai --- applications/wg-easy/docs/development-workflow.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/applications/wg-easy/docs/development-workflow.md b/applications/wg-easy/docs/development-workflow.md index 8516e95d..4a1ca689 100644 --- a/applications/wg-easy/docs/development-workflow.md +++ b/applications/wg-easy/docs/development-workflow.md @@ -96,14 +96,11 @@ Configure chart values and create or modify templates. > Tools required by tasks in this project will be made available in a container. Run the commands below to start the dev environment ``` -# Build tools image. Run this command once. -task dev:build-image +# Open shell to execute tasks +task dev:shell # Start/restart tools container. Idempotent. task dev:restart - -# Open shell to execute tasks -task dev:shell ``` Validate chart templates locally without deploying to a cluster.