upgrade tests: add CSI workload

Add an upgrade test workload for CSI with the AWS EBS plugin. In order to
validate this workload, we'll need to deploy the plugin jobs and then register
volumes with them. So this extends the `run_workloads` module to allow for a
"post script" to be run after a given job has been deployed. We can use that as
a model for other test workloads.

Ref: https://hashicorp.atlassian.net/browse/NET-12217

enos/enos-scenario-upgrade.hcl

@@ -76,6 +76,7 @@ scenario "upgrade" {
       consul_license            = var.consul_license
       volumes                   = false
       region                    = var.aws_region
+      availability_zone         = var.aws_az
       instance_arch             = matrix.arch
     }
   }
@@ -89,18 +90,35 @@ scenario "upgrade" {
 
     module = module.run_workloads
     variables {
-      nomad_addr  = step.provision_cluster.nomad_addr
-      ca_file     = step.provision_cluster.ca_file
-      cert_file   = step.provision_cluster.cert_file
-      key_file    = step.provision_cluster.key_file
-      nomad_token = step.provision_cluster.nomad_token
+      nomad_addr   = step.provision_cluster.nomad_addr
+      ca_file      = step.provision_cluster.ca_file
+      cert_file    = step.provision_cluster.cert_file
+      key_file     = step.provision_cluster.key_file
+      nomad_token  = step.provision_cluster.nomad_token
+      availability_zone         = var.aws_az
+
       workloads = {
+        # deploy these first
+        csi_plugin_ebs_controller = {
+          job_spec    = "jobs/plugin-aws-ebs-controller.nomad.hcl"
+          alloc_count = 2
+          type        = "service"
+        }
+        csi_plugin_ebs_node = {
+          job_spec    = "jobs/plugin-aws-ebs-nodes.nomad.hcl"
+          alloc_count = 0
+          type        = "system"
+          post_script = "scripts/wait_for_ebs_plugin.sh"
+        }
+
         service_raw_exec = { job_spec = "jobs/raw-exec-service.nomad.hcl", alloc_count = 3, type = "service" }
         service_docker   = { job_spec = "jobs/docker-service.nomad.hcl", alloc_count = 3, type = "service" }
         system_docker    = { job_spec = "jobs/docker-system.nomad.hcl", alloc_count = 0, type = "system" }
         batch_docker     = { job_spec = "jobs/docker-batch.nomad.hcl", alloc_count = 3, type = "batch" }
         batch_raw_exec   = { job_spec = "jobs/raw-exec-batch.nomad.hcl", alloc_count = 3, type = "batch" }
         system_raw_exec  = { job_spec = "jobs/raw-exec-system.nomad.hcl", alloc_count = 0, type = "system" }
+        wants_csi        = { job_spec = "jobs/wants-volume.nomad.hcl", alloc_count = 1, type = "service" }
+
       }
     }
 

enos/enos-vars.hcl

@@ -54,3 +54,8 @@ variable "aws_region" {
   description = "The AWS region to deploy to."
   default     = "us-east-1"
 }
+
+variable "aws_az" {
+  description = "The AWS availability zone to deploy to."
+  default     = "us-east-1b"
+}

enos/modules/run_workloads/jobs/plugin-aws-ebs-controller.nomad.hcl

@@ -0,0 +1,45 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+variable "alloc_count" {
+  type    = number
+  default = 2
+}
+
+job "plugin-aws-ebs-controller" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "controller" {
+
+    count = var.alloc_count
+
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0"
+
+        args = [
+          "controller",
+          "--endpoint=${CSI_ENDPOINT}",
+          "--logtostderr",
+          "--v=5",
+        ]
+      }
+
+      csi_plugin {
+        id        = "aws-ebs0"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 256
+      }
+    }
+  }
+}

enos/modules/run_workloads/jobs/plugin-aws-ebs-nodes.nomad.hcl

@@ -0,0 +1,48 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# this variable is not used but required by runner
+variable "alloc_count" {
+  type    = number
+  default = 1
+}
+
+job "plugin-aws-ebs-nodes" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  type = "system"
+
+  group "nodes" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.33.0"
+
+        args = [
+          "node",
+          "--endpoint=${CSI_ENDPOINT}",
+          "--logtostderr",
+          "--v=5",
+        ]
+
+        privileged = true
+      }
+
+      csi_plugin {
+        id        = "aws-ebs0"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 100
+        memory = 256
+      }
+    }
+  }
+}

enos/modules/run_workloads/jobs/raw-exec-system.nomad.hcl

@@ -1,6 +1,7 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
+# this variable is not used but required by runner
 variable "alloc_count" {
   type    = number
   default = 1
@@ -24,7 +25,7 @@ job "system-raw-exec" {
 #!/bin/bash
 
 while true; do
-    sleep 30000  
+    sleep 30000
 done
 EOH
         destination = "local/runme.sh"

enos/modules/run_workloads/jobs/wants-volume.nomad.hcl

@@ -0,0 +1,75 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "alloc_count" {
+  type    = number
+  default = 1
+}
+
+# a job that mounts an EBS volume and writes its job ID as a file
+job "wants-ebs-volume" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "group" {
+    count = var.alloc_count
+
+    volume "test" {
+      type            = "csi"
+      source          = "ebsTestVolume"
+      attachment_mode = "file-system"
+      access_mode     = "single-node-writer"
+    }
+
+    task "task" {
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "httpd"
+        args    = ["-vv", "-f", "-p", "8001", "-h", "/local"]
+      }
+
+      volume_mount {
+        volume      = "test"
+        destination = "${NOMAD_TASK_DIR}/test"
+        read_only   = false
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+
+    task "sidecar" {
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "echo '${NOMAD_ALLOC_ID}' > ${NOMAD_TASK_DIR}/index.html"]
+      }
+
+      lifecycle {
+        hook    = "poststart"
+        sidecar = false
+      }
+
+      volume_mount {
+        volume      = "test"
+        destination = "${NOMAD_TASK_DIR}/test"
+        read_only   = false
+      }
+
+      resources {
+        cpu    = 10
+        memory = 10
+      }
+
+    }
+  }
+}

enos/modules/run_workloads/main.tf

@@ -9,48 +9,72 @@ terraform {
   }
 }
 
+resource "random_pet" "volume_tag" {
+}
+
 locals {
   nomad_env = {
     NOMAD_ADDR        = var.nomad_addr
     NOMAD_CACERT      = var.ca_file
     NOMAD_CLIENT_CERT = var.cert_file
     NOMAD_CLIENT_KEY  = var.key_file
     NOMAD_TOKEN       = var.nomad_token
+    VOLUME_TAG      = random_pet.volume_tag.id
   }
 
   system_job_count     = length({ for k, v in var.workloads : k => v if v.type == "system" })
   service_batch_allocs = sum([for wl in var.workloads : wl.alloc_count])
 }
 
+# test volume we'll register for the CSI workload
+resource "aws_ebs_volume" "test_volume" {
+  availability_zone = var.availability_zone
+  size              = 10
+
+  tags = {
+    VolumeTag = random_pet.volume_tag.id
+  }
+}
+
 resource "enos_local_exec" "wait_for_nomad_api" {
   environment = local.nomad_env
 
   scripts = [abspath("${path.module}/scripts/wait_for_nomad_api.sh")]
 }
 
 resource "enos_local_exec" "get_nodes" {
+  depends_on = [enos_local_exec.wait_for_nomad_api]
   environment = local.nomad_env
 
   inline = ["nomad node status -json | jq '[.[] | select(.Status == \"ready\")] | length'"]
 }
 
 resource "enos_local_exec" "get_jobs" {
+  depends_on = [enos_local_exec.wait_for_nomad_api]
   environment = local.nomad_env
 
   inline = ["nomad job status| awk '$4 == \"running\" {count++} END {print count+0}'"]
 }
 
 resource "enos_local_exec" "get_allocs" {
+  depends_on = [enos_local_exec.wait_for_nomad_api]
   environment = local.nomad_env
 
   inline = ["nomad alloc status -json | jq '[.[] | select(.ClientStatus == \"running\")] | length'"]
 }
 
 resource "enos_local_exec" "workloads" {
-  depends_on = [enos_local_exec.get_jobs, enos_local_exec.get_allocs]
-  for_each   = var.workloads
+  depends_on = [
+    enos_local_exec.get_jobs,
+    enos_local_exec.get_allocs,
+    aws_ebs_volume.test_volume
+  ]
+  for_each = var.workloads
 
   environment = local.nomad_env
 
-  inline = ["nomad job run -var alloc_count=${each.value.alloc_count} ${path.module}/${each.value.job_spec}"]
+  inline = [
+    "nomad job run -var alloc_count=${each.value.alloc_count} ${path.module}/${each.value.job_spec}",
+    each.value.post_script != null ? abspath("${path.module}/${each.value.post_script}") : "echo ok"
+  ]
 }

enos/modules/run_workloads/scripts/volume.hcl.tpl

@@ -0,0 +1,13 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+type        = "csi"
+id          = "ebsTestVolume"
+name        = "IDEMPOTENCY_TOKEN"
+external_id = "EXTERNAL_ID"
+plugin_id   = "aws-ebs0"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}