WIP tests/e2e: Add encrypted image test for operator

.github/workflows/ccruntime_e2e.yaml

@@ -55,6 +55,13 @@ jobs:
         env:
           TARGET_BRANCH: ${{ inputs.target-branch }}
 
+      - name: Login to Confidential Containers ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Install deps
         run: |
           sudo apt-get update -y

tests/e2e/ansible/group_vars/all

@@ -15,6 +15,8 @@ build_pkgs:
     - gcc
 container_runtime: containerd
 go_version: 1.22.6
+rust_version: 1.75.0
+skopeo_version: f64a376
 # conntrack and socat are needed by the `kubeadm init` preflight checks
 kubeadm_pkgs:
   ubuntu:

tests/e2e/ansible/install_test_deps.yaml

@@ -33,6 +33,49 @@
                 path: bats-core
                 state: absent
           when: bats_exist.rc != 0
+    - block:
+      - name: Check skopeo is installed
+        shell: command -v skopeo >/dev/null 2>&1
+        register: skopeo_exist
+        ignore_errors: yes
+      - name: Install skopeo
+        shell: |
+          sudo apt-get install -y build-essential libgpgme-dev libassuan-dev libbtrfs-dev pkg-config go-md2man
+          git clone https://github.com/containers/skopeo
+          cd skopeo
+          git checkout {{ skopeo_version }}
+          DISABLE_DOCS=1 make
+          DISABLE_DOCS=1 sudo make install
+    - block:
+      - name: Install kbs test dependencies
+        shell: |
+          sudo apt-get install -y build-essential pkg-config libssl-dev
+    - block:
+      - name: Check rust is installed
+        shell: command -v rustc >/dev/null 2>&1
+        register: rustc_exist
+        ignore_errors: yes
+      - name: Install rust
+        shell: |
+          curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain {{ rust_version }}
+    - block:
+      - name: Create temp trustee directory
+        shell: |
+              mkdir /tmp/trustee
+      - name: Clone trustee repository
+        git:
+          repo: https://github.com/confidential-containers/trustee.git
+          dest: /tmp/trustee
+          clone: yes
+          force: yes
+        retries: 3
+        delay: 10
+      - name: Build and install the kbs client
+        shell: |
+          export PATH="${PATH}:${HOME}/.cargo/bin"
+          cd /tmp/trustee/kbs
+          make CLI_FEATURES=sample_only cli
+          sudo make install-cli
     - block:
       - name: Download Go tarball
         get_url:
@@ -52,6 +95,19 @@
           src: /usr/local/go/bin/go
           dest: /usr/local/bin/go
           state: link
+    - block:
+      - name: Check yq is installed
+        shell: command -v yq >/dev/null 2>&1
+        register: yq_exist
+        ignore_errors: yes
+      - name: Install yq
+        shell: |
+          export yq_pkg="github.com/mikefarah/yq"
+          export yq_version=v4.40.7
+          export yq_path="/usr/local/bin/yq"
+          yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_linux_{{ target_arch }}"
+          curl -o "${yq_path}" -LSsf "${yq_url}"
+          chmod +x "${yq_path}"
     - block:
       - name: Check kustomize is installed
         shell: command -v kustomize >/dev/null 2>&1
@@ -81,3 +137,14 @@
             - /usr/local/go
             - /usr/local/bin/bats
             - /usr/local/bin/kustomize
+        - name: Uninstall kbs-client
+          shell: |
+            cd /tmp/trustee/kbs
+            sudo make uninstall
+        - name: Remove temporary file
+          file:
+            path: "{{ item }}"
+            state: absent
+          with_items:
+            - /tmp/trustee
+

tests/e2e/encrypted.sh

@@ -0,0 +1,162 @@
+#!/bin/bash
+#
+# Copyright Confidential Containers Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+source "${script_dir}/lib.sh"
+
+export key_file="${key_file:-image_key}"
+export key_path="${key_path:-/default/image_key/nginx}"
+export trustee_repo="${trustee_repo:-https://github.com/confidential-containers/trustee.git}"
+export kbs_namespace="${kbs_namespace:-coco-tenant}"
+export aa_kbc="${aa_kbc:-cc_kbc}"
+export kbs_svc_name="${kbs_svc_name:-kbs}"
+export kbs_ingress_name="${kbs_ingress_name:-kbs}"
+export RUNTIMECLASS="${RUNTIMECLASS:-}"
+export username="${username:-}"
+export encrypted_image="${encrypted_image:-ghcr.io/$username/nginx:encrypted}"
+
+build_encryption_key() {
+	head -c 32 /dev/urandom | openssl enc > "${key_file}"
+	mkdir output
+	docker run -v "$PWD/output:/output" ghcr.io/confidential-containers/staged-images/coco-keyprovider:075b9a9ee77227d9d92b6f3649ef69de5e72d204 /encrypt.sh \
+		-k "$(base64 < ${key_file})" \
+		-i kbs:///some/key/id \
+		-s docker://nginx:stable \
+		-d dir:/output
+
+	skopeo copy dir:output "docker://${encrypted_image}"
+	skopeo inspect "docker://${encrypted_image}" | jq -r '.LayersData[].MIMEType' | grep encrypted
+}
+
+deploy_k8s_kbs() {
+	if [ ! -d "${script_dir}/trustee" ]; then
+		git clone "${trustee_repo}"
+	fi
+
+	pushd "${script_dir}/trustee/kbs/config/kubernetes"
+		cat "${script_dir}/${key_file}" > overlays/key.bin
+		export DEPLOYMENT_DIR=nodeport
+		./deploy-kbs.sh
+
+		kbs_pod=$(kubectl -n "${kbs_namespace}" get pods -o NAME)
+		kubectl rollout status -w --timeout=30s deployment/kbs -n "${kbs_namespace}"
+	popd
+}
+
+delete_k8s_kbs() {
+	pushd "${script_dir}/trustee/kbs/config/kubernetes"
+		kubectl delete -k overlays/
+	popd
+	rm -rf "${script_dir}/trustee"
+}
+
+provide_image_key() {
+	kubectl exec -n "${kbs_namespace}" "${kbs_pod}" -- mkdir -p "/opt/confidential-containers/kbs/repository/$(dirname "$key_path")"
+	cat "${script_dir}/${key_file}" | kubectl exec -i -n "${kbs_namespace}" "${kbs_pod}" -- tee "/opt/confidential-containers/kbs/repository/${key_path}" > /dev/null
+}
+
+launch_pod() {
+	kubectl apply -f "${script_dir}/nginx-encrypted.yaml"
+	kubectl rollout status -w --timeout=30s deployment/nginx-encrypted
+}
+
+delete_pod() {
+	kubectl delete -f "${script_dir}/nginx-encrypted.yaml"
+}
+
+check_image_key() {
+	kubectl logs -n "${kbs_namespace}" "${kbs_pod}" | grep "${key_path}"
+}
+
+set_metadata_annotation() {
+	local yaml="${1}"
+	local key="${2}"
+	local value="${3}"
+	local metadata_path="${4:-}"
+	local annotation_key=""
+
+ 	[ -n "$metadata_path" ] && annotation_key+="${metadata_path}."
+
+ 	# yaml annotation key name.
+ 	annotation_key+="metadata.annotations.\"${key}\""
+
+ 	echo "$annotation_key"
+	# yq set annotations in yaml. Quoting the key because it can have
+	# dots.
+	yq -i ".${annotation_key} = \"${value}\"" "${yaml}"
+}
+
+set_aa_kbc() {
+	local cc_kbs_addr
+	export cc_kbs_addr=$(kbs_k8s_svc_http_addr)
+	kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params"
+	kernel_params_value="agent.guest_components_rest_api=resource"
+ 	if [ "${aa_kbc}" = "cc_kbc" ]; then
+		kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${cc_kbs_addr}"
+	fi
+	set_metadata_annotation "${script_dir}/nginx-encrypted.yaml" \
+		"${kernel_params_annotation}" \
+		"${kernel_params_value}"
+}
+
+kbs_k8s_svc_http_addr() {
+	local host
+	local port
+
+ 	host=$(kbs_k8s_svc_host)
+	port=$(kbs_k8s_svc_port)
+
+	echo "http://${host}:${port}"
+}
+
+kbs_k8s_svc_host() {
+	if kubectl get ingress -n "${kbs_namespace}" 2>/dev/null | grep -q kbs; then
+		kubectl get ingress "${kbs_ingress_name}" -n "${kbs_namespace}" \
+			-o jsonpath='{.spec.rules[0].host}' 2>/dev/null
+	elif kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" &>/dev/null; then
+			local host
+			host=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' -n "${kbs_namespace}")
+			echo "$host"
+	else
+		kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" \
+			-o jsonpath='{.spec.clusterIP}' 2>/dev/null
+	fi
+}
+
+kbs_k8s_svc_port() {
+	if kubectl get ingress -n "${kbs_namespace}" 2>/dev/null | grep -q kbs; then
+		echo "80"
+	elif kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" &>/dev/null; then
+		kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" -o jsonpath='{.spec.ports[0].nodePort}'
+	else
+		kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" \
+			-o jsonpath='{.spec.ports[0].port}' 2>/dev/null
+	fi
+}
+
+set_parameters() {
+	sudo sed -i "s/runtimeclass/${RUNTIMECLASS}/g" "${script_dir}/nginx-encrypted.yaml"
+	sudo sed -i "s/user/${username}/g" "${script_dir}/nginx-encrypted.yaml"
+}
+
+main() {
+	build_encryption_key
+	deploy_k8s_kbs
+	set_aa_kbc
+ 	set_parameters
+	provide_image_key
+	launch_pod
+	check_image_key
+	delete_k8s_kbs
+ 	delete_pod
+}
+
+main "$@"

tests/e2e/nginx-encrypted.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: nginx
+  name: nginx-encrypted
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+      annotations:
+        io.containerd.cri.runtime-handler: runtimeclass
+    spec:
+      runtimeClassName: runtimeclass
+      containers:
+        - image: ghcr.io/user/nginx:encrypted
+          name: nginx

tests/e2e/operator_tests.bats

@@ -21,6 +21,12 @@ setup() {
 	ns="confidential-containers-system"
 }
 
+@test "$test_tag Test encrypt image" {
+is_operator_installed
+
+"${BATS_TEST_DIRNAME}/encrypted.sh"
+}
+
 @test "$test_tag Test can uninstall the operator" {
 
 # Assume the operator is installed, otherwise fail.