upgrade tests: add CSI workload

Add an upgrade test workload for CSI with the AWS EBS plugin. In order to
validate this workload, we'll need to deploy the plugin jobs and then create
volumes with them. So this extends the `run_workloads` module to allow for a
"wait script" to be run after a given job has been deployed. We can use that as
a model for other test workloads.

Ref: https://hashicorp.atlassian.net/browse/NET-12217

enos/modules/run_workloads/jobs/plugin-aws-ebs-controller.nomad.hcl

@@ -0,0 +1,45 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+variable "alloc_count" {
+  type    = number
+  default = 2
+}
+
+job "plugin-aws-ebs-controller" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "controller" {
+
+    count = var.alloc_count
+
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0"
+
+        args = [
+          "controller",
+          "--endpoint=${CSI_ENDPOINT}",
+          "--logtostderr",
+          "--v=5",
+        ]
+      }
+
+      csi_plugin {
+        id        = "aws-ebs0"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 256
+      }
+    }
+  }
+}

enos/modules/run_workloads/jobs/plugin-aws-ebs-nodes.nomad.hcl

@@ -0,0 +1,42 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "plugin-aws-ebs-nodes" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  type = "system"
+
+  group "nodes" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0"
+
+        args = [
+          "node",
+          "--endpoint=${CSI_ENDPOINT}",
+          "--logtostderr",
+          "--v=5",
+        ]
+
+        privileged = true
+      }
+
+      csi_plugin {
+        id        = "aws-ebs0"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 256
+      }
+    }
+  }
+}

enos/modules/run_workloads/main.tf

@@ -28,5 +28,8 @@ resource "enos_local_exec" "workloads" {
 
   environment = local.nomad_env
 
-  inline = ["nomad job run -var alloc_count=${each.value.alloc_count} ${path.module}/${each.value.job_spec}"]
+  inline = [
+    "nomad job run -var alloc_count=${each.value.alloc_count} ${path.module}/${each.value.job_spec}",
+    each.value.wait_script != "" ? each.value.wait_script : "echo ok"
+  ]
 }

enos/modules/run_workloads/scripts/volume.hcl.tpl

@@ -0,0 +1,34 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+id        = "VOLUME_ID"
+name      = "IDEMPOTENCY_TOKEN" # CSIVolumeName tag, must be unique and idempotent
+type      = "csi"
+plugin_id = "aws-ebs0"
+
+capacity_min = "10GiB"
+capacity_max = "20GiB"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "block-device"
+}
+
+parameters {
+  type = "gp2"
+}
+
+topology_request {
+  required {
+    topology {
+      segments {
+        "topology.ebs.csi.aws.com/zone" = "AWS_ZONE"
+      }
+    }
+  }
+}

enos/modules/run_workloads/scripts/wait_for_ebs_plugin.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+set -euo pipefail
+
+# note: it can a very long time for plugins to come up
+TIMEOUT=60
+INTERVAL=2
+last_error=
+start_time=$(date +%s)
+
+checkPlugin() {
+    local pluginStatus foundControllers foundNodes
+    pluginStatus=$(nomad plugin status aws-ebs0) || {
+        last_error="could not read CSI plugin status"
+        return 1
+    }
+
+    foundControllers=$(echo "$pluginStatus" | awk -F'= +' '/Controllers Healthy/{print $2}')
+    if [[ "$foundControllers" != 2 ]]; then
+        echo "$foundControllers"
+        last_error="expected plugin to have 2 healthy controllers, found $foundControllers"
+        return 1
+    fi
+
+    foundNodes=$(echo "$pluginStatus" | awk -F'= +' '/Nodes Healthy/{print $2}')
+    if [[ "$foundNodes" == 0 ]]; then
+        last_error="expected plugin to have at least 1 healthy nodes, found none"
+        return 1
+    fi
+    return 0
+}
+
+awsZone=
+
+findZone() {
+    local firstNodeID
+    firstNodeID=$(nomad node status -json | jq -r '.[0].ID') || {
+        echo "could not read node status"
+        exit 1
+    }
+    awsZone=$(nomad node status -json "$firstNodeID" | jq -r '.Attributes."platform.aws.placement.availability-zone"') || {
+        echo "could not read AWS zone from node status"
+        exit 1
+    }
+}
+
+createVolume() {
+    local volumeID idempotencyToken
+    volumeID=$1
+    idempotencyToken=$(uuidgen)
+
+    sed -e "s/VOLUME_ID/$volumeID/" \
+        -e "s/IDEMPOTENCY_TOKEN/$idempotencyToken/" \
+        -e "s/AWS_ZONE/$awsZone/" \
+        volume.hcl.tpl | nomad volume create -
+}
+
+while :
+do
+    checkPlugin && break
+
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+    if [ "$elapsed_time" -ge "$TIMEOUT" ]; then
+        echo "Error: CSI plugin did not become available within $TIMEOUT seconds."
+        exit 1
+    fi
+
+    sleep "$INTERVAL"
+done
+
+findZone
+createVolume ebsVolume0
+createVolume ebsVolume1
+
+nomad volume status -type csi

enos/modules/run_workloads/variables.tf

@@ -34,10 +34,16 @@ variable "workloads" {
   type = map(object({
     job_spec    = string
     alloc_count = number
+    wait_script = string
   }))
 
   default = {
+    # deploy these first
+    csi_plugin_ebs_controller = { job_spec = "jobs/plugin-aws-ebs-controller.nomad.hcl", alloc_count = 2}
+    csi_plugin_ebs_node = { job_spec = "jobs/plugin-aws-ebs-controller.nomad.hcl", alloc_count = 0, wait_scipt="scripts/wait_for_ebs_plugin.sh" }
+
     service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3 }
     service_docker   = { job_spec = "jobs/docker-service.nomad.hcl", alloc_count = 3 }
+
   }
 }