Merge pull request #7215 from freedomofpress/stg-gcloud

Use upstream gcloud-sdk image instead of our oudated one

Makefile

@@ -603,15 +603,6 @@ ci-deb-tests:  ## Test SecureDrop Debian packages in CI environment.
 	@$(SDROOT)/devops/scripts/test-built-packages.sh
 	@echo
 
-.PHONY: build-gcloud-docker
-build-gcloud-docker:  ## Build Docker container for Google Cloud SDK.
-	@echo "Building Docker container for Google Cloud SDK..."
-	@echo "${GCLOUD_VERSION}" > devops/gce-nested/gcloud-container.ver && \
-	@docker build --build-arg="GCLOUD_VERSION=${GCLOUD_VERSION}" \
-				 -f devops/docker/Dockerfile.gcloud \
-				 -t "quay.io/freedomofpress/gcloud-sdk:${GCLOUD_VERSION}" .
-	@echo
-
 .PHONY: vagrant-package
 vagrant-package:  ## Package a Vagrant box of the last stable SecureDrop release.
 	@echo "███ Packaging Vagrant box of last stable SecureDrop release."

devops/gce-nested/ci-env.sh

@@ -17,7 +17,6 @@ export BUILD_NUM="${CIRCLE_BUILD_NUM}"
 export PROJECT_ID="securedrop-ci"
 export JOB_NAME="sd-ci-nested"
 export GCLOUD_MACHINE_TYPE="c2-standard-8"
-GCLOUD_CONTAINER_VER="$(cat "${TOPLEVEL}/devops/gce-nested/gcloud-container.ver")"
 export GCLOUD_CONTAINER_VER
 export CLOUDSDK_COMPUTE_ZONE="us-west1-c"
 export EPHEMERAL_DIRECTORY="/tmp/gce-nested"
@@ -30,6 +29,7 @@ export SSH_PUBKEY="${SSH_PRIVKEY}.pub"
 # retrievable via GOOGLE_CREDENTIALS. Let's read that value, decode it,
 # and write it to disk in the CI environment so the gcloud tooling
 # can authenticate.
+# This function is called multiple times and must be idempotent.
 function generate_gce_creds_file() {
     # First check if there is an existing cred file
     if [ ! -f "${GCE_CREDS_FILE}" ]; then
@@ -48,26 +48,25 @@ function generate_gce_creds_file() {
             fi
         fi
     fi
-}
 
-# Wrapper function to communicate with the gcloud API. Ensure gcloud-sdk
-# container is running, and if so, pass all args to it.
-function gcloud_call() {
-    if ! (docker ps | grep -q gcloud_tool); then
-        docker run --rm \
-                --env="CLOUDSDK_COMPUTE_ZONE=${CLOUDSDK_COMPUTE_ZONE}" \
-                --volume "${EPHEMERAL_DIRECTORY}/gce.pub:/gce.pub" \
-                --volume "${GCE_CREDS_FILE}:/gce-svc-acct.json" \
-                --name gcloud_tool -d \
-                "quay.io/freedomofpress/gcloud-sdk:${GCLOUD_CONTAINER_VER}" \
-                background >/dev/null 2>&1
-        # Give container a moment for gcloud tooling to authenticate
-        # Kept falling over on first calls without this
-        sleep 3
+    # Prime the "gcloud-config" container by authenticating from $GCE_CREDS_FILE.
+    if ! docker inspect --format '{{ .Id }}' gcloud-config > /dev/null 2>&1; then
+        docker run \
+            --env="CLOUDSDK_COMPUTE_ZONE=${CLOUDSDK_COMPUTE_ZONE}" \
+            --volume "${GCE_CREDS_FILE}:/gce-svc-acct.json" \
+            --name gcloud-config \
+            gcr.io/google.com/cloudsdktool/google-cloud-cli:stable \
+            gcloud auth activate-service-account --key-file /gce-svc-acct.json
     fi
+}
 
-    docker exec -i gcloud_tool \
-        /usr/bin/gcloud --project "${PROJECT_ID}" "$@"
+# Each gcloud_call() invocation just runs a fresh (note "--rm") instance of the "google-cloud-cli" container primed from "gcloud-config" above.
+function gcloud_call() {
+    docker run --rm \
+        --volumes-from gcloud-config \
+        --env="CLOUDSDK_COMPUTE_ZONE=${CLOUDSDK_COMPUTE_ZONE}" \
+        gcr.io/google.com/cloudsdktool/google-cloud-cli:stable \
+        gcloud --project "${PROJECT_ID}" "$@"
 }
 
 

devops/gce-nested/ci-go.sh

@@ -14,12 +14,6 @@ set -o pipefail
 
 export BASE_OS="${BASE_OS:-focal}"
 
-# Temporary workaround for old gcloud-sdk image
-sudo mkdir -p /etc/systemd/system/docker.service.d
-echo -e "[Service]\nEnvironment=\"DOCKER_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=true\"" | sudo tee -a /etc/systemd/system/docker.service.d/env.conf
-sudo systemctl daemon-reload
-sudo systemctl restart docker
-
 ./devops/gce-nested/gce-start.sh
 ./devops/gce-nested/gce-runner.sh
 ./devops/gce-nested/gce-stop.sh

devops/scripts/boot-strap-venv.sh

@@ -51,7 +51,13 @@ function virtualenv_bootstrap() {
         then
             p=$(command -v "python${PYTHON_VERSION}" 2> /dev/null || command -v python3)
             echo "Creating ${p} virtualenv in ${VENV}"
-            "${p}" -m venv "${VENV}"
+            # be flexible in venv creation, e.g. staging has virtualenv while
+            # deb-tests (GHA runner) has python3-venv
+            if command -v virtualenv > /dev/null; then
+                virtualenv -p "${p}" "${VENV}"
+            else
+                "${p}" -m venv "${VENV}"
+            fi
         fi
 
         PIP_CONSTRAINT=${DEV_CONSTRAINT} "${VENV}/bin/pip" install -q -r "securedrop/requirements/python3/develop-requirements.txt"