From 9ba0cb6604d1a6ee3c405a7de8f56a8f5afc3159 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 8 Aug 2023 15:17:07 -0700
Subject: [PATCH 01/57] create sps api url ssm param

---
 .../terraform-unity-sps-hysds-cluster/sps_api.tf      | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
index 10fe4aa..41f02ba 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
@@ -20,17 +20,10 @@ resource "kubernetes_service" "sps-api-service" {
 }
 
 resource "aws_ssm_parameter" "sps-api-hostname-param" {
-  name        = "/unity/sps/${var.deployment_name}/spsApi/hostname"
+  name        = "/unity/sps/${var.deployment_name}/spsApi/url"
   description = "Hostname of sps api load balancer"
   type        = "String"
-  value       = kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname
-}
-
-resource "aws_ssm_parameter" "sps-api-port-param" {
-  name        = "/unity/sps/${var.deployment_name}/spsApi/port"
-  description = "Port used by sps api"
-  type        = "String"
-  value       = var.service_port_map.sps_api_service
+  value       = "http://${kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.sps_api_service}"
 }
 
 resource "kubernetes_deployment" "sps-api" {

From 38dfd1eff4cad33c404d6b08c3ba8aec46b77899 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 14 Aug 2023 11:28:51 -0700
Subject: [PATCH 02/57] hysds elasticsearch deployments default private

---
 .../terraform-unity-sps-hysds-cluster/elastic.tf              | 4 ++--
 .../terraform-unity-sps-hysds-cluster/outputs.tf              | 4 ++--
 terraform-unity/outputs.tf                                    | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
index c8ab348..11ff692 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -129,7 +129,7 @@ locals {
     clusterHealthCheckParams = "wait_for_status=yellow&timeout=1s"
     replicas                 = 1
     service = {
-      type = var.service_type
+      type = "NodePort"
       port = var.service_port_map.mozart_es
     }
     httpPort      = var.service_port_map.mozart_es
@@ -239,7 +239,7 @@ locals {
     clusterHealthCheckParams = "wait_for_status=yellow&timeout=1s"
     replicas                 = 1
     service = {
-      type = var.service_type
+      type = "NodePort"
       port = var.service_port_map.grq2_es
     }
     httpPort      = var.service_port_map.grq2_es
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
index c45d379..6e707c9 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
@@ -22,8 +22,8 @@ data "kubernetes_service" "jobs-es" {
 output "load_balancer_hostnames" {
   description = "Load Balancer Ingress Hostnames"
   value = {
-    mozart_es     = data.kubernetes_service.mozart-es.status[0].load_balancer[0].ingress[0].hostname,
-    grq_es        = data.kubernetes_service.grq-es.status[0].load_balancer[0].ingress[0].hostname,
+    # mozart_es     = data.kubernetes_service.mozart-es.status[0].load_balancer[0].ingress[0].hostname,
+    # grq_es        = data.kubernetes_service.grq-es.status[0].load_balancer[0].ingress[0].hostname,
     jobs_es       = data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname,
     ades_wpst_api = kubernetes_service.ades-wpst-api-service.status[0].load_balancer[0].ingress[0].hostname,
     # ades_wpst_api = aws_elb.ades_wpst_api_elb.dns_name
diff --git a/terraform-unity/outputs.tf b/terraform-unity/outputs.tf
index b8f2f81..bf164aa 100644
--- a/terraform-unity/outputs.tf
+++ b/terraform-unity/outputs.tf
@@ -1,8 +1,8 @@
 output "load_balancer_hostnames" {
   description = "Load Balancer Ingress Hostnames"
   value = {
-    mozart_es = module.unity-sps-hysds-cluster.load_balancer_hostnames.mozart_es
-    grq_es    = module.unity-sps-hysds-cluster.load_balancer_hostnames.grq_es
+    # mozart_es = module.unity-sps-hysds-cluster.load_balancer_hostnames.mozart_es
+    # grq_es    = module.unity-sps-hysds-cluster.load_balancer_hostnames.grq_es
     jobs_es   = module.unity-sps-hysds-cluster.load_balancer_hostnames.jobs_es
     ades_wpst = module.unity-sps-hysds-cluster.load_balancer_hostnames.ades_wpst_api
     sps_api   = module.unity-sps-hysds-cluster.load_balancer_hostnames.sps_api

From edbf02b706fcd81fe71f3786c17367c1de2768f5 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 14 Aug 2023 11:29:47 -0700
Subject: [PATCH 03/57] use dev releases of wpst and sps_hysds_pge_base

---
 .../terraform-unity-sps-hysds-cluster/variables.tf            | 4 ++--
 terraform-unity/variables.tf                                  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
index 105334c..d49699a 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -63,9 +63,9 @@ variable "docker_images" {
     hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
     hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
     hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
-    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.1.0"
+    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"
     rabbitmq           = "rabbitmq:3.11.13-management"
     busybox            = "busybox:1.36.0"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index 1a1b504..fd70fdf 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -64,9 +64,9 @@ variable "docker_images" {
     hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
     hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
     hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
-    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.1.0"
+    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"
     rabbitmq           = "rabbitmq:3.11.13-management"
     busybox            = "busybox:1.36.0"

From 95c53f57eb435a95d77528b2946c90eb6655b4f4 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Tue, 15 Aug 2023 16:53:23 -0700
Subject: [PATCH 04/57] Add tags to ELBs and update docs

---
 .github/ISSUE_TEMPLATE/feature_request.md     |  2 +-
 .github/PULL_REQUEST_TEMPLATE.md              |  2 +-
 CHANGELOG.md                                  |  3 +-
 README.md                                     |  2 +-
 hysds/configs/.netrc                          |  2 +-
 .../README.md                                 | 37 ++++++++++++++++--
 .../ades_wpst.tf                              |  7 +++-
 .../api_gateway.tf                            |  2 +-
 .../terraform-unity-sps-hysds-cluster/efs.tf  |  2 +-
 .../elastic.tf                                | 38 +++++++++----------
 .../mozart.tf                                 |  3 --
 .../outputs.tf                                |  3 +-
 .../sps_api.tf                                |  6 +++
 terraform-unity/README.md                     |  7 ++--
 terraform-unity/main.tf                       |  2 +-
 terraform-unity/ssm_params.tf                 |  2 +-
 .../features/delete_prewarm_request.feature   |  2 +-
 .../get_prewarm_request_status.feature        |  2 +-
 unity-test/features/start_prewarm.feature     |  2 +-
 unity-test/requirements.txt                   |  4 +-
 20 files changed, 83 insertions(+), 47 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 0b74a7a..05a0a1c 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -29,6 +29,6 @@ Ex. I'm frustrated when [...] happens as documented in issue-XYZ
 
 **Describe the feature request**
 
-> A clear and concise description of your request. 
+> A clear and concise description of your request.
 
 Ex. I need or want [...]
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 8ef73bb..543fb3e 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -8,6 +8,6 @@
 - Links to relevant issues
 - Example: issue-XYZ
 ## Testing
-- Provide some proof you've tested your changes 
+- Provide some proof you've tested your changes
 - Example: test results available at ...
 - Example: tested on operating system ...
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ecf4858..f8bb8d4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,9 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.1.0] - 2022-04-20
 
-### Added 
+### Added
 
 - First implementation of HySDS services as Docker containers
 - Shell scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using the kubectl client
 - Terraform scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using Terraform
-
diff --git a/README.md b/README.md
index 24dbf02..24856d6 100644
--- a/README.md
+++ b/README.md
@@ -1 +1 @@
-# unity-sps-prototype
\ No newline at end of file
+# unity-sps-prototype
diff --git a/hysds/configs/.netrc b/hysds/configs/.netrc
index cc12757..9a15de2 100644
--- a/hysds/configs/.netrc
+++ b/hysds/configs/.netrc
@@ -1,3 +1,3 @@
 machine rabbitmq
 login guest
-password guest
\ No newline at end of file
+password guest
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/README.md b/terraform-modules/terraform-unity-sps-hysds-cluster/README.md
index 798ff45..cb8f1a5 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/README.md
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/README.md
@@ -35,21 +35,42 @@ No modules.
 | [aws_api_gateway_method.api_gateway_ades_wpst_proxy_method](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/api_gateway_method) | resource |
 | [aws_api_gateway_resource.api_gateway_ades_wpst_proxy_resource](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/api_gateway_resource) | resource |
 | [aws_api_gateway_resource.api_gateway_ades_wpst_resource](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/api_gateway_resource) | resource |
+| [aws_efs_file_system.verdi-stage-efs](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/efs_file_system) | resource |
+| [aws_efs_mount_target.verdi-stage-efs-mnt-target](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/efs_mount_target) | resource |
 | [aws_eks_node_group.sps_api](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/eks_node_group) | resource |
 | [aws_eks_node_group.verdi](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/eks_node_group) | resource |
+| [aws_iam_policy.cloudwatch_logs_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.eks_sps_api_policy](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.sqs_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_policy) | resource |
+| [aws_iam_policy_attachment.cloudwatch_logs_access_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_policy_attachment) | resource |
+| [aws_iam_policy_attachment.sqs_access_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_policy_attachment) | resource |
 | [aws_iam_role.eks_sps_api_node_role](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role) | resource |
 | [aws_iam_role.eks_verdi_node_role](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role) | resource |
+| [aws_iam_role.lambda_role](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.eks_sps_api_node_group_scaling_policy](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_sps_api_node_role_customer_policies](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_sps_api_node_role_managed_policies](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_verdi_node_role_customer_policies](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_verdi_node_role_managed_policies](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.eks_verdi_node_role_sns_policy](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_lambda_event_source_mapping.sqs_event_source_mapping_jobs_data_ingest](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/lambda_event_source_mapping) | resource |
+| [aws_lambda_function.jobs_data_ingest](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/lambda_function) | resource |
+| [aws_launch_template.sps_api_node_group_launch_template](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/launch_template) | resource |
+| [aws_launch_template.verdi_node_group_launch_template](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/launch_template) | resource |
+| [aws_security_group.verdi-efs-sg](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/security_group) | resource |
 | [aws_security_group_rule.efs_egress](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/security_group_rule) | resource |
 | [aws_security_group_rule.efs_ingress](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/security_group_rule) | resource |
+| [aws_security_group_rule.verdi_efs_egress](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/security_group_rule) | resource |
+| [aws_security_group_rule.verdi_efs_ingress](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/security_group_rule) | resource |
+| [aws_sns_topic.jobs_data](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sns_topic) | resource |
+| [aws_sns_topic_policy.jobs_data](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sns_topic_policy) | resource |
+| [aws_sns_topic_subscription.sqs_subscription](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sns_topic_subscription) | resource |
+| [aws_sqs_queue.jobs_data_ingest_queue](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sqs_queue) | resource |
+| [aws_sqs_queue_policy.jobs_data_queue](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sqs_queue_policy) | resource |
 | [aws_ssm_parameter.sps-api-hostname-param](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/ssm_parameter) | resource |
 | [aws_ssm_parameter.sps-api-port-param](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/ssm_parameter) | resource |
 | [helm_release.grq2-es](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
+| [helm_release.jobs-es](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
 | [helm_release.mozart-es](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
 | [kubernetes_config_map.celeryconfig](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/config_map) | resource |
 | [kubernetes_config_map.cwl-workflow-utils](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/config_map) | resource |
@@ -77,10 +98,13 @@ No modules.
 | [kubernetes_namespace.unity-sps](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/namespace) | resource |
 | [kubernetes_persistent_volume.ades-wpst-sqlite-pv](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume) | resource |
 | [kubernetes_persistent_volume.grq-es-pv](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume) | resource |
+| [kubernetes_persistent_volume.jobs-es-pv](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume) | resource |
 | [kubernetes_persistent_volume.mozart-es-pv](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume) | resource |
 | [kubernetes_persistent_volume.uads-development-efs](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume) | resource |
+| [kubernetes_persistent_volume.verdi-stage-efs-pv](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume) | resource |
 | [kubernetes_persistent_volume_claim.ades-wpst-sqlite-pv-claim](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume_claim) | resource |
 | [kubernetes_persistent_volume_claim.uads-development-efs](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume_claim) | resource |
+| [kubernetes_persistent_volume_claim.verdi-worker-pv-claim](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/persistent_volume_claim) | resource |
 | [kubernetes_role.verdi-reader](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/role) | resource |
 | [kubernetes_role_binding.verdi-reader-binding](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/role_binding) | resource |
 | [kubernetes_secret.sps-api](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/secret) | resource |
@@ -96,17 +120,21 @@ No modules.
 | [kubernetes_stateful_set.rabbitmq_statefulset](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/stateful_set) | resource |
 | [kubernetes_storage_class.efs_storage_class](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/resources/storage_class) | resource |
 | [null_resource.api_gateway_stage_update_resource](https://registry.terraform.io/providers/hashicorp/null/3.1.1/docs/resources/resource) | resource |
+| [null_resource.build_lambda_package](https://registry.terraform.io/providers/hashicorp/null/3.1.1/docs/resources/resource) | resource |
+| [null_resource.upload_jobs_template](https://registry.terraform.io/providers/hashicorp/null/3.1.1/docs/resources/resource) | resource |
 | [random_id.counter](https://registry.terraform.io/providers/hashicorp/random/3.4.3/docs/resources/id) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/caller_identity) | data source |
 | [aws_efs_mount_target.uads-development-efs-fsmt](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/efs_mount_target) | data source |
 | [aws_eks_cluster.sps-cluster](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/eks_cluster) | data source |
 | [aws_launch_template.default_group_node_group](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/launch_template) | data source |
+| [aws_security_groups.sps-cluster-sg](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/security_groups) | data source |
 | [aws_ssm_parameter.api_gateway_rest_api_id](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.api_gateway_rest_api_lambda_authorizer_id](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.api_gateway_rest_api_root_resource_id](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_subnets.eks_subnets](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/subnets) | data source |
 | [aws_vpc.eks_vpc](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/vpc) | data source |
 | [kubernetes_service.grq-es](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/data-sources/service) | data source |
+| [kubernetes_service.jobs-es](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/data-sources/service) | data source |
 | [kubernetes_service.mozart-es](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/data-sources/service) | data source |
 
 ## Inputs
@@ -122,18 +150,19 @@ No modules.
 | <a name="input_counter"></a> [counter](#input\_counter) | value | `string` | `""` | no |
 | <a name="input_datasets_filename"></a> [datasets\_filename](#input\_datasets\_filename) | value | `string` | `"datasets.remote.template.json"` | no |
 | <a name="input_default_group_node_group_launch_template_name"></a> [default\_group\_node\_group\_launch\_template\_name](#input\_default\_group\_node\_group\_launch\_template\_name) | value | `string` | n/a | yes |
+| <a name="input_default_group_node_group_name"></a> [default\_group\_node\_group\_name](#input\_default\_group\_node\_group\_name) | value | `string` | `"defaultgroupNodeGroup"` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Unique name of this deployment in the account. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.0.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.0.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.0.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.0.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.0.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.0.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.0.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.0.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.0.0"<br>}</pre> | no |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.1.0"<br>}</pre> | no |
 | <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | value | `string` | n/a | yes |
-| <a name="input_eks_node_groups"></a> [eks\_node\_groups](#input\_eks\_node\_groups) | value | `string` | n/a | yes |
 | <a name="input_elb_subnets"></a> [elb\_subnets](#input\_elb\_subnets) | value | `string` | n/a | yes |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes |
+| <a name="input_mcp_linux_eks_optimized_ami"></a> [mcp\_linux\_eks\_optimized\_ami](#input\_mcp\_linux\_eks\_optimized\_ami) | value | `string` | `"ami-04db7a1ae7708642e"` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace for the Unity SPS HySDS-related Kubernetes resources | `string` | `"unity-sps"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
 | <a name="input_region"></a> [region](#input\_region) | The AWS region | `string` | `"us-west-2"` | no |
 | <a name="input_release"></a> [release](#input\_release) | The SPS release version | `string` | n/a | yes |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
-| <a name="input_service_port_map"></a> [service\_port\_map](#input\_service\_port\_map) | value | `map(number)` | <pre>{<br>  "ades_wpst_api_service": 5001,<br>  "grq2_es": 9201,<br>  "grq2_service": 8878,<br>  "hysds_ui_service": 3000,<br>  "mozart_es": 9200,<br>  "mozart_service": 8888,<br>  "rabbitmq_mgmt_service_cluster_rpc": 15672,<br>  "rabbitmq_service_cluster_rpc": 15672,<br>  "rabbitmq_service_epmd": 4369,<br>  "rabbitmq_service_listener": 5672,<br>  "redis_service": 6379,<br>  "sps_api_service": 5002<br>}</pre> | no |
+| <a name="input_service_port_map"></a> [service\_port\_map](#input\_service\_port\_map) | value | `map(number)` | <pre>{<br>  "ades_wpst_api_service": 5001,<br>  "grq2_es": 9201,<br>  "grq2_service": 8878,<br>  "hysds_ui_service": 3000,<br>  "jobs_es": 9202,<br>  "mozart_es": 9200,<br>  "mozart_service": 8888,<br>  "rabbitmq_mgmt_service_cluster_rpc": 15672,<br>  "rabbitmq_service_cluster_rpc": 15672,<br>  "rabbitmq_service_epmd": 4369,<br>  "rabbitmq_service_listener": 5672,<br>  "redis_service": 6379,<br>  "sps_api_service": 5002<br>}</pre> | no |
 | <a name="input_service_type"></a> [service\_type](#input\_service\_type) | value | `string` | `"LoadBalancer"` | no |
 | <a name="input_uads_development_efs_fsmt_id"></a> [uads\_development\_efs\_fsmt\_id](#input\_uads\_development\_efs\_fsmt\_id) | value | `string` | `null` | no |
 | <a name="input_uds_client_id"></a> [uds\_client\_id](#input\_uds\_client\_id) | value | `string` | n/a | yes |
@@ -149,4 +178,4 @@ No modules.
 | Name | Description |
 |------|-------------|
 | <a name="output_load_balancer_hostnames"></a> [load\_balancer\_hostnames](#output\_load\_balancer\_hostnames) | Load Balancer Ingress Hostnames |
-<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
\ No newline at end of file
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 7562c57..819252d 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -88,8 +88,13 @@ resource "kubernetes_service" "ades-wpst-api-service" {
   metadata {
     name      = "ades-wpst-api"
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
-    # todo remove when testing sepereate elb
     annotations = {
+      "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-adeswpst-RestApiLoadBalancer-${local.counter}"
+      "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
+        "Name"      = "${var.project}-${var.venue}-${var.service_area}-adeswpst-RestApiLoadBalancer-${local.counter}"
+        "Component" = "adeswpst"
+        "Stack"     = "adeswpst"
+      }) : format("%s=%s", k, v)])
       "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
     }
   }
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/api_gateway.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
index 78ea14c..38dbcbb 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
@@ -138,4 +138,4 @@ resource "null_resource" "api_gateway_stage_update_resource" {
   provisioner "local-exec" {
     command = "aws apigateway update-stage --region ${var.region} --rest-api-id ${data.aws_ssm_parameter.api_gateway_rest_api_id[0].value} --stage-name=${var.venue} --patch-operations op='replace',path='/deploymentId',value='${aws_api_gateway_deployment.api_gateway_deployment[0].id}'"
   }
-}
\ No newline at end of file
+}
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/efs.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/efs.tf
index 81ffdaa..ecab830 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/efs.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/efs.tf
@@ -142,4 +142,4 @@ resource "aws_security_group_rule" "verdi_efs_egress" {
   protocol                 = "-1"
   security_group_id        = aws_security_group.verdi-efs-sg.id
   source_security_group_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].cluster_security_group_id
-}
\ No newline at end of file
+}
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
index c8ab348..1fad97d 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -417,13 +417,13 @@ resource "helm_release" "mozart-es" {
     yamlencode({
       "service" = {
         "annotations" = {
+          "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-mozart-ElasticsearchLoadBalancer-${local.counter}"
+          "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
+            "Name"      = "${var.project}-${var.venue}-${var.service_area}-mozart-ElasticsearchLoadBalancer-${local.counter}"
+            "Component" = "mozart"
+            "Stack"     = "mozart"
+          }) : format("%s=%s", k, v)])
           "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-          "service.beta.kubernetes.io/aws-load-balancer-name"    = "${var.project}-${var.venue}-${var.service_area}-mozart-LoadBalancer-${local.counter}"
-          "service.beta.kubernetes.io/aws-load-balancer-tags" = jsonencode(merge(local.common_tags, {
-            Name      = "${var.project}-${var.venue}-${var.service_area}-mozart-LoadBalancer-${local.counter}"
-            Component = "mozart"
-            Stack     = "mozart"
-          }))
         }
       }
     })
@@ -443,13 +443,13 @@ resource "helm_release" "grq2-es" {
     yamlencode({
       "service" = {
         "annotations" = {
+          "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-grq-ElasticsearchLoadBalancer-${local.counter}"
+          "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
+            "Name"      = "${var.project}-${var.venue}-${var.service_area}-grq-ElasticsearchLoadBalancer-${local.counter}"
+            "Component" = "grq"
+            "Stack"     = "grq"
+          }) : format("%s=%s", k, v)])
           "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-          "service.beta.kubernetes.io/aws-load-balancer-name"    = "${var.project}-${var.venue}-${var.service_area}-GRQ-LoadBalancer-${local.counter}"
-          "service.beta.kubernetes.io/aws-load-balancer-tags" = jsonencode(merge(local.common_tags, {
-            Name      = "${var.project}-${var.venue}-${var.service_area}-GRQ-LoadBalancer-${local.counter}"
-            Component = "GRQ"
-            Stack     = "GRQ"
-          }))
         }
       }
     })
@@ -469,15 +469,15 @@ resource "helm_release" "jobs-es" {
     yamlencode({
       "service" = {
         "annotations" = {
+          "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-jobs-ElasticsearchLoadBalancer-${local.counter}"
+          "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
+            "Name"      = "${var.project}-${var.venue}-${var.service_area}-jobs-ElasticsearchLoadBalancer-${local.counter}"
+            "Component" = "jobs"
+            "Stack"     = "jobs"
+          }) : format("%s=%s", k, v)])
           "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-          "service.beta.kubernetes.io/aws-load-balancer-name"    = "${var.project}-${var.venue}-${var.service_area}-jobs-LoadBalancer-${local.counter}"
-          "service.beta.kubernetes.io/aws-load-balancer-tags" = jsonencode(merge(local.common_tags, {
-            Name      = "${var.project}-${var.venue}-${var.service_area}-jobs-LoadBalancer-${local.counter}"
-            Component = "jobs"
-            Stack     = "jobs"
-          }))
         }
       }
     })
   ]
-}
\ No newline at end of file
+}
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/mozart.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/mozart.tf
index 5b918d5..b1b48b3 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/mozart.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/mozart.tf
@@ -3,7 +3,6 @@ resource "kubernetes_service" "mozart-service" {
     name      = "mozart"
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
-
   spec {
     selector = {
       app = "mozart"
@@ -15,8 +14,6 @@ resource "kubernetes_service" "mozart-service" {
   }
 }
 
-
-
 resource "kubernetes_deployment" "mozart" {
   metadata {
     name      = "mozart"
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
index c45d379..342bcfd 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
@@ -26,7 +26,6 @@ output "load_balancer_hostnames" {
     grq_es        = data.kubernetes_service.grq-es.status[0].load_balancer[0].ingress[0].hostname,
     jobs_es       = data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname,
     ades_wpst_api = kubernetes_service.ades-wpst-api-service.status[0].load_balancer[0].ingress[0].hostname,
-    # ades_wpst_api = aws_elb.ades_wpst_api_elb.dns_name
-    sps_api = kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname,
+    sps_api       = kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname,
   }
 }
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
index 10fe4aa..89ecaed 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
@@ -3,6 +3,12 @@ resource "kubernetes_service" "sps-api-service" {
     name      = "sps-api"
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
     annotations = {
+      "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-spsapi-RestApiLoadBalancer-${local.counter}"
+      "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
+        "Name"      = "${var.project}-${var.venue}-${var.service_area}-spsapi-RestApiLoadBalancer-${local.counter}"
+        "Component" = "spsapi"
+        "Stack"     = "spsapi"
+      }) : format("%s=%s", k, v)])
       "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
     }
   }
diff --git a/terraform-unity/README.md b/terraform-unity/README.md
index 8546ab2..7b7b3f8 100644
--- a/terraform-unity/README.md
+++ b/terraform-unity/README.md
@@ -211,16 +211,17 @@ $ rm terraform.tf.backup
 | <a name="input_default_group_node_group_launch_template_name"></a> [default\_group\_node\_group\_launch\_template\_name](#input\_default\_group\_node\_group\_launch\_template\_name) | value | `string` | `null` | no |
 | <a name="input_default_group_node_group_name"></a> [default\_group\_node\_group\_name](#input\_default\_group\_node\_group\_name) | value | `string` | `null` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Unique name of this deployment in the account. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.0.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.0.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.0.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.0.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.0.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.0.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.0.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.0.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.0.0"<br>}</pre> | no |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.1.0"<br>}</pre> | no |
 | <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | The name of the EKS cluster. | `string` | n/a | yes |
 | <a name="input_elb_subnets"></a> [elb\_subnets](#input\_elb\_subnets) | value | `string` | `null` | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes |
+| <a name="input_mcp_linux_eks_optimized_ami"></a> [mcp\_linux\_eks\_optimized\_ami](#input\_mcp\_linux\_eks\_optimized\_ami) | value | `string` | `"ami-04db7a1ae7708642e"` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace for the Unity SPS HySDS-related Kubernetes resources | `string` | `"unity-sps"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `null` | no |
 | <a name="input_region"></a> [region](#input\_region) | The AWS region | `string` | `"us-west-2"` | no |
 | <a name="input_release"></a> [release](#input\_release) | The SPS release version | `string` | n/a | yes |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
-| <a name="input_service_port_map"></a> [service\_port\_map](#input\_service\_port\_map) | value | `map(number)` | <pre>{<br>  "ades_wpst_api_service": 5001,<br>  "grq2_es": 9201,<br>  "grq2_service": 8878,<br>  "hysds_ui_service": 3000,<br>  "mozart_es": 9200,<br>  "mozart_service": 8888,<br>  "rabbitmq_mgmt_service_cluster_rpc": 15672,<br>  "rabbitmq_service_cluster_rpc": 15672,<br>  "rabbitmq_service_epmd": 4369,<br>  "rabbitmq_service_listener": 5672,<br>  "redis_service": 6379,<br>  "sps_api_service": 5002<br>}</pre> | no |
+| <a name="input_service_port_map"></a> [service\_port\_map](#input\_service\_port\_map) | value | `map(number)` | <pre>{<br>  "ades_wpst_api_service": 5001,<br>  "grq2_es": 9201,<br>  "grq2_service": 8878,<br>  "hysds_ui_service": 3000,<br>  "jobs_es": 9202,<br>  "mozart_es": 9200,<br>  "mozart_service": 8888,<br>  "rabbitmq_mgmt_service_cluster_rpc": 15672,<br>  "rabbitmq_service_cluster_rpc": 15672,<br>  "rabbitmq_service_epmd": 4369,<br>  "rabbitmq_service_listener": 5672,<br>  "redis_service": 6379,<br>  "sps_api_service": 5002<br>}</pre> | no |
 | <a name="input_service_type"></a> [service\_type](#input\_service\_type) | value | `string` | `"LoadBalancer"` | no |
 | <a name="input_uads_development_efs_fsmt_id"></a> [uads\_development\_efs\_fsmt\_id](#input\_uads\_development\_efs\_fsmt\_id) | value | `string` | `null` | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | `null` | no |
@@ -233,4 +234,4 @@ $ rm terraform.tf.backup
 | Name | Description |
 |------|-------------|
 | <a name="output_load_balancer_hostnames"></a> [load\_balancer\_hostnames](#output\_load\_balancer\_hostnames) | Load Balancer Ingress Hostnames |
-<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
\ No newline at end of file
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 6047354..4a622be 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -37,4 +37,4 @@ module "unity-sps-hysds-cluster" {
   verdi_node_group_instance_types               = var.verdi_node_group_instance_types
   mcp_linux_eks_optimized_ami                   = var.mcp_linux_eks_optimized_ami
   add_routes_to_api_gateway                     = var.add_routes_to_api_gateway
-}
\ No newline at end of file
+}
diff --git a/terraform-unity/ssm_params.tf b/terraform-unity/ssm_params.tf
index d378b09..b49449f 100644
--- a/terraform-unity/ssm_params.tf
+++ b/terraform-unity/ssm_params.tf
@@ -39,4 +39,4 @@ data "aws_ssm_parameter" "account_venue" {
 data "aws_ssm_parameter" "account_project" {
   count = var.project == null ? 1 : 0
   name  = "/unity/account/project"
-}
\ No newline at end of file
+}
diff --git a/unity-test/features/delete_prewarm_request.feature b/unity-test/features/delete_prewarm_request.feature
index 53d7e05..456bdb6 100644
--- a/unity-test/features/delete_prewarm_request.feature
+++ b/unity-test/features/delete_prewarm_request.feature
@@ -4,4 +4,4 @@ Feature: Delete SPS Prewarm Request
         When a DELETE request is called on the SPS API prewarm request endpoint
         Then the HTTP response contains a successful status code
         And the HTTP response body contains a request id
-        And the prewarm request is deleted
\ No newline at end of file
+        And the prewarm request is deleted
diff --git a/unity-test/features/get_prewarm_request_status.feature b/unity-test/features/get_prewarm_request_status.feature
index 39cac7a..9c1cd82 100644
--- a/unity-test/features/get_prewarm_request_status.feature
+++ b/unity-test/features/get_prewarm_request_status.feature
@@ -3,4 +3,4 @@ Feature: Get SPS Prewarm Request Status
         Given the prewarm request has been created
         When a GET request is called on the SPS API prewarm request endpoint
         Then the HTTP response contains a successful status code
-        And the HTTP response body contains a request id
\ No newline at end of file
+        And the HTTP response body contains a request id
diff --git a/unity-test/features/start_prewarm.feature b/unity-test/features/start_prewarm.feature
index 9d0f7dc..f35b21c 100644
--- a/unity-test/features/start_prewarm.feature
+++ b/unity-test/features/start_prewarm.feature
@@ -3,4 +3,4 @@ Feature: Start SPS Prewarming
         Given the proper JSON data for the POST request body
         When a POST request is called on the SPS API prewarm endpoint
         Then the HTTP response contains a successful status code
-        And the HTTP response body contains a request id
\ No newline at end of file
+        And the HTTP response body contains a request id
diff --git a/unity-test/requirements.txt b/unity-test/requirements.txt
index 285a948..f29b1b1 100644
--- a/unity-test/requirements.txt
+++ b/unity-test/requirements.txt
@@ -1,5 +1,5 @@
+backoff>=2.2.1
+pathlib>=1.0.1
 pytest-bdd>=6.1.1
 pytest-dotenv>=0.5.2
 requests>=2.28.2
-pathlib>=1.0.1
-backoff>=2.2.1

From add845fc0ae8ea2efa88ba6f08c81f2758401fdb Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 15 Aug 2023 17:30:34 -0700
Subject: [PATCH 05/57] add tests for jobs database to integration test suite

---
 .../features/jobs_database_status.feature     |  48 +++++++
 unity-test/requirements.txt                   |   1 +
 unity-test/step_defs/conftest.py              |  19 +++
 .../step_defs/test_job_database_status.py     | 127 ++++++++++++++++++
 4 files changed, 195 insertions(+)
 create mode 100644 unity-test/features/jobs_database_status.feature
 create mode 100644 unity-test/step_defs/test_job_database_status.py

diff --git a/unity-test/features/jobs_database_status.feature b/unity-test/features/jobs_database_status.feature
new file mode 100644
index 0000000..42b67a5
--- /dev/null
+++ b/unity-test/features/jobs_database_status.feature
@@ -0,0 +1,48 @@
+Feature: The Jobs Database has Job Status
+    Scenario Outline: Following a job execution request of a deployed algorithm process, the job is added to the jobs database
+        Given the <project_name> <process_name> algorithm process has been deployed to the ADES
+        And a WPS-T request is made to execute the process
+        And the HTTP response contains a status code of 201
+        And the response includes a Location header
+        And the Location header contains a job ID
+        When the status of the job is queried through the jobs database
+        Then the job status is "submitted" or "failed"
+        And the job has the request data in appropriate fields
+
+        Examples:
+        | project_name | process_name     |
+        | sounder_sips |  L1A             |
+        | sounder_sips |  L1B             |
+        | sounder_sips |  chirp |
+
+    Scenario Outline: Following a job execution request of a deployed algorithm process that results in a success, the jobs database reflects the success
+        Given the <project_name> <process_name> algorithm process has been deployed to the ADES
+        And a WPS-T request is made to execute the process
+        And the HTTP response contains a status code of 201
+        And the response includes a Location header
+        And the Location header contains a job ID
+        And the job runs successfully
+        When the status of the job is queried through the jobs database
+        Then the jobs status is "succeeded"
+
+        Examples:
+        | project_name | process_name     |
+        | sounder_sips |  L1A             |
+        | sounder_sips |  L1B             |
+        | sounder_sips |  chirp |
+
+    Scenario Outline: Following a job execution request of a deployed algorithm process, the jobs database show that the job is running
+        Given the <project_name> <process_name> algorithm process has been deployed to the ADES
+        And a WPS-T request is made to execute the process
+        And the HTTP response contains a status code of 201
+        And the response includes a Location header
+        And the Location header contains a job ID
+        And the job is running
+        When the status of the job is queried through the jobs database
+        Then the job status is "running"
+
+        Examples:
+        | project_name | process_name     |
+        | sounder_sips |  L1A             |
+        | sounder_sips |  L1B             |
+        | sounder_sips |  chirp |
\ No newline at end of file
diff --git a/unity-test/requirements.txt b/unity-test/requirements.txt
index 285a948..485b6d6 100644
--- a/unity-test/requirements.txt
+++ b/unity-test/requirements.txt
@@ -3,3 +3,4 @@ pytest-dotenv>=0.5.2
 requests>=2.28.2
 pathlib>=1.0.1
 backoff>=2.2.1
+elasticsearch==7.17.9
diff --git a/unity-test/step_defs/conftest.py b/unity-test/step_defs/conftest.py
index 998a34b..71c0548 100644
--- a/unity-test/step_defs/conftest.py
+++ b/unity-test/step_defs/conftest.py
@@ -1,5 +1,6 @@
 import pytest
 from pytest_bdd import given, then, parsers
+from elasticsearch import Elasticsearch
 import requests
 from urllib.parse import urljoin
 import re
@@ -40,12 +41,26 @@ def process_service_endpoint(request):
 def sps_api_service_endpoint(request):
     return request.config.getoption("--sps-api-service-endpoint")
 
+@pytest.fixture()
+def jobs_database_endpoint(request):
+    return request.config.getoption("--job-database-endpoint")
+
+@pytest.fixture()
+def jobs_database_client(jobs_database_endpoint):
+    return Elasticsearch(jobs_database_endpoint)
 
 @pytest.fixture
 def projects():
     data = reader.request_body("", "", reader.projects)
     return data
 
+@pytest.fixture
+def job_request_body(project_process_dict):
+    return reader.request_body(
+        project_process_dict["project_name"],
+        project_process_dict["process_name"],
+        reader.execution_post_request_body,
+    )
 
 @pytest.fixture
 def user_selected_processes(request, projects):
@@ -238,3 +253,7 @@ def location_header_contains_job_id(location_header):
     job_id = location_header.rsplit("/jobs/", 1)[-1]
     assert job_id
     return job_id
+
+@given("the HTTP response contains a status code of 201")
+def created_response(response):
+    assert response.status_code == 201
\ No newline at end of file
diff --git a/unity-test/step_defs/test_job_database_status.py b/unity-test/step_defs/test_job_database_status.py
new file mode 100644
index 0000000..bd24cea
--- /dev/null
+++ b/unity-test/step_defs/test_job_database_status.py
@@ -0,0 +1,127 @@
+from pytest_bdd import scenario, given, when, then, parsers
+from elasticsearch import Elasticsearch
+import backoff
+import requests
+from .conftest import FEATURES_DIR, _request_job_status_by_id
+from .utils import JsonReader
+
+feature_file = "jobs_database_status.feature"
+FEATURE_FILE = FEATURES_DIR.joinpath(feature_file)
+
+
+@scenario(
+    FEATURE_FILE,
+    "Following a job execution request of a deployed algorithm process, the job is added to the jobs database",
+)
+def test_following_processing_request_job_added_to_jobs_database():
+    pass
+
+@scenario(
+    FEATURE_FILE,
+    "Following a job execution request of a deployed algorithm process that results in a success, the jobs database reflects the success",
+)
+def test_following_successful_processing_request_job_added_to_job_database_status_is_success():
+    pass
+
+@scenario(
+    FEATURE_FILE,
+    "Following a job execution request of a deployed algorithm process, the jobs database show that the job is running",
+)
+def test_following_processing_request_job_added_to_jobs_database():
+    pass
+
+def fatal_status(e):
+    fatal = False
+    if isinstance(e, AssertionError):
+        if "failed" in e.args[0]:
+            fatal = True
+    return fatal
+
+@given(
+    "the job runs successfully"
+)
+@backoff.on_exception(
+    backoff.constant,
+    (AssertionError, requests.exceptions.HTTPError),
+    max_time=3600,
+    giveup=fatal_status,
+    jitter=None,
+    interval=0.1, # check more frequently since jobs can execute quickly
+)
+def request_job_status_by_id_running(process_service_endpoint, project_process_dict, job_id):
+    job_status_response = _request_job_status_by_id(
+        process_service_endpoint,
+        project_process_dict["process_name"],
+        job_id,
+    )
+    response_json = job_status_response.json()
+    assert "status" in response_json
+
+    job_status = response_json["status"]
+    assert job_status == "running"
+
+
+@given(
+    "the job runs successfully"
+)
+@backoff.on_exception(
+    backoff.constant,
+    (AssertionError, requests.exceptions.HTTPError),
+    max_time=3600,
+    giveup=fatal_status,
+    jitter=None,
+    interval=1,
+)
+def request_job_status_by_id_succeeded(process_service_endpoint, project_process_dict, job_id):
+    job_status_response = _request_job_status_by_id(
+        process_service_endpoint,
+        project_process_dict["process_name"],
+        job_id,
+    )
+    response_json = job_status_response.json()
+    assert "status" in response_json
+
+    job_status = response_json["status"]
+    assert job_status == "succeeded"
+
+@when(
+    "the status of the job is monitored through the jobs database",
+    target_fixture="job_from_database"
+)
+def request_job_by_id_jobs_database(jobs_database_client, job_id):
+    job = jobs_database_client.get(index="jobs", id=job_id)
+    return job["_source"]
+
+@then(
+    'the job status is "submitted" or "failed"'
+)
+def job_status_submitted_or_failed(job_from_database):
+    assert job_from_database["status"] == "submitted" or job_from_database["status"] == "failed"
+
+@then(
+    'the job has the request data in appropriate fields'
+)
+def job_data_in_appropriate_fields(job_from_database, job_request_body):
+    fields = ["id", "status", "inputs", "outputs", "labels"]
+    for field in fields:
+        assert field in job_from_database
+    
+    for input in job_request_body["inputs"]:
+        assert input["id"] in job_from_database["inputs"]
+        assert job_from_database["inputs"][input["id"]] == input["data"]
+    
+    if "labels" in job_request_body:
+        for label in job_request_body["labels"]:
+            assert label in job_from_database["labels"]
+
+@then(
+    'the job status is "succeeded"'
+)
+def job_status_submitted_or_failed(job_from_database):
+    assert job_from_database["status"] == "succeeded"
+
+@then(
+    'the job status is "running"'
+)
+def job_status_submitted_or_failed(job_from_database):
+    assert job_from_database["status"] == "running"
\ No newline at end of file

From 46f011a18fdc6b21e66e3423e492d52e452c354b Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 16 Aug 2023 13:32:03 -0700
Subject: [PATCH 06/57] add job_db_test options

---
 unity-test/step_defs/conftest.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/unity-test/step_defs/conftest.py b/unity-test/step_defs/conftest.py
index 71c0548..e086a24 100644
--- a/unity-test/step_defs/conftest.py
+++ b/unity-test/step_defs/conftest.py
@@ -24,6 +24,12 @@ def pytest_addoption(parser):
         action="store",
         help="Base URL for the SPS API service endpoint",
     )
+    parser.addoption(
+        "--jobs-database-endpoint",
+        action="store",
+        help="Base URL for the Jobs Database endpoint",
+        required=True,
+    )
     parser.addoption(
         "--sounder-sips-process-selection",
         action="store",
@@ -31,7 +37,6 @@ def pytest_addoption(parser):
         required=True,
     )
 
-
 @pytest.fixture(scope="module", autouse=True)
 def process_service_endpoint(request):
     return request.config.getoption("--process-service-endpoint")
@@ -43,7 +48,7 @@ def sps_api_service_endpoint(request):
 
 @pytest.fixture()
 def jobs_database_endpoint(request):
-    return request.config.getoption("--job-database-endpoint")
+    return request.config.getoption("--jobs-database-endpoint")
 
 @pytest.fixture()
 def jobs_database_client(jobs_database_endpoint):

From a7af14456fae389178c308d47eb5018f923443fd Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 16 Aug 2023 13:42:28 -0700
Subject: [PATCH 07/57] add jobs db test to mcp test regression test workflow

---
 .github/workflows/regression_test_mcp_test.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/regression_test_mcp_test.yml b/.github/workflows/regression_test_mcp_test.yml
index af4243e..34f83ca 100644
--- a/.github/workflows/regression_test_mcp_test.yml
+++ b/.github/workflows/regression_test_mcp_test.yml
@@ -13,6 +13,9 @@ on:
       SPS_API_SERVICE_ENDPOINT:
         description: "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number)"
         type: string
+      JOBS_DATABASE_ENDPOINT:
+        description: "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number)"
+        type: string
       SOUNDER_SIPS_PROCESS_SELECTION:
         description: "The Sounder SIPS process to test (L1A, L1B, chirp)"
         default: ".*"
@@ -26,6 +29,7 @@ jobs:
       - run: |
           echo "Base URL for the WPS-T API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_TEST_PROCESS_SERVICE_ENDPOINT }}"
           echo "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_TEST_SPS_API_SERVICE_ENDPOINT }}"
+          echo "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}"
           echo "The Sounder SIPS PGEs to test (L1A, L1B, chirp): ${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}"
   regression_test:
     runs-on: ubuntu-latest
@@ -46,5 +50,6 @@ jobs:
           pytest -s
           --process-service-endpoint=${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_TEST_PROCESS_SERVICE_ENDPOINT }}
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_TEST_SPS_API_SERVICE_ENDPOINT }}
+          --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
           -rs

From 9d18431dbf9a6cfc9e47af0bb33bb1e0e3a70cfc Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 16 Aug 2023 13:48:25 -0700
Subject: [PATCH 08/57] add jobs db test to mcp dev regression test workflow

---
 .github/workflows/regression_test_mcp_dev.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/regression_test_mcp_dev.yml b/.github/workflows/regression_test_mcp_dev.yml
index a175ea9..8ff3bf9 100644
--- a/.github/workflows/regression_test_mcp_dev.yml
+++ b/.github/workflows/regression_test_mcp_dev.yml
@@ -13,6 +13,9 @@ on:
       SPS_API_SERVICE_ENDPOINT:
         description: "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number)"
         type: string
+      JOBS_DATABASE_ENDPOINT:
+        description: "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number)"
+        type: string
       SOUNDER_SIPS_PROCESS_SELECTION:
         description: "The Sounder SIPS process to test (L1A, L1B, chirp)"
         default: ".*"
@@ -26,6 +29,7 @@ jobs:
       - run: |
           echo "Base URL for the WPS-T API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_DEV_PROCESS_SERVICE_ENDPOINT }}"
           echo "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_DEV_SPS_API_SERVICE_ENDPOINT }}"
+          echo "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_DEV_JOBS_DATABASE_ENDPOINT }}"
           echo "The Sounder SIPS PGEs to test (L1A, L1B, chirp): ${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}"
   regression_test:
     runs-on: ubuntu-latest
@@ -46,5 +50,6 @@ jobs:
           pytest -s
           --process-service-endpoint=${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_DEV_PROCESS_SERVICE_ENDPOINT }}
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_DEV_SPS_API_SERVICE_ENDPOINT }}
+          --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
           -rs

From 41845e85cd885243767cf6b44590c840618468b1 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 16 Aug 2023 16:12:12 -0700
Subject: [PATCH 09/57] fix name mismatch in jobs db tests

---
 unity-test/step_defs/test_job_database_status.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/unity-test/step_defs/test_job_database_status.py b/unity-test/step_defs/test_job_database_status.py
index bd24cea..17baad7 100644
--- a/unity-test/step_defs/test_job_database_status.py
+++ b/unity-test/step_defs/test_job_database_status.py
@@ -38,7 +38,7 @@ def fatal_status(e):
     return fatal
 
 @given(
-    "the job runs successfully"
+    "the job is running"
 )
 @backoff.on_exception(
     backoff.constant,
@@ -85,7 +85,7 @@ def request_job_status_by_id_succeeded(process_service_endpoint, project_process
     assert job_status == "succeeded"
 
 @when(
-    "the status of the job is monitored through the jobs database",
+    "the status of the job is queried through the jobs database",
     target_fixture="job_from_database"
 )
 def request_job_by_id_jobs_database(jobs_database_client, job_id):

From 7a3a1b82d14a5e1769e5482e0bdf0de7639f10bb Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Fri, 18 Aug 2023 16:46:38 -0700
Subject: [PATCH 10/57] modify job tests for package restrictions

---
 .../sleep/deploy_post_request_body.json       | 30 +++++++++++++++++++
 .../sleep/execution_post_request_body.json    | 12 ++++++++
 .../features/jobs_database_status.feature     | 28 ++---------------
 .../step_defs/test_job_database_status.py     | 16 +++++-----
 4 files changed, 54 insertions(+), 32 deletions(-)
 create mode 100644 unity-test/data/test_workflows/sleep/deploy_post_request_body.json
 create mode 100644 unity-test/data/test_workflows/sleep/execution_post_request_body.json

diff --git a/unity-test/data/test_workflows/sleep/deploy_post_request_body.json b/unity-test/data/test_workflows/sleep/deploy_post_request_body.json
new file mode 100644
index 0000000..210f0d3
--- /dev/null
+++ b/unity-test/data/test_workflows/sleep/deploy_post_request_body.json
@@ -0,0 +1,30 @@
+{
+  "processDescription": {
+    "process": {
+      "id": "sleep",
+      "title": "Sleep Job",
+      "owsContext": {
+        "offering": {
+          "code": "http://www.opengis.net/eoc/applicationContext/cwl",
+          "content": {
+            "href": "https://raw.githubusercontent.com/unity-sds/unity-sps-workflows/main/sounder_sips/utils/sleep_job_wrapper.cwl"
+          }
+        }
+      },
+      "abstract": "sleep",
+      "keywords": [],
+      "inputs": [],
+      "outputs": []
+    },
+    "processVersion": "develop",
+    "jobControlOptions": ["async-execute"],
+    "outputTransmission": ["reference"]
+  },
+  "immediateDeployment": true,
+  "executionUnit": [
+    {
+      "href": "docker.registry/l1b-cwl:latest"
+    }
+  ],
+  "deploymentProfileName": "http://www.opengis.net/profiles/eoc/dockerizedApplication"
+}
\ No newline at end of file
diff --git a/unity-test/data/test_workflows/sleep/execution_post_request_body.json b/unity-test/data/test_workflows/sleep/execution_post_request_body.json
new file mode 100644
index 0000000..06fd252
--- /dev/null
+++ b/unity-test/data/test_workflows/sleep/execution_post_request_body.json
@@ -0,0 +1,12 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ],
+  "labels": ["test", "sleep"]
+}
\ No newline at end of file
diff --git a/unity-test/features/jobs_database_status.feature b/unity-test/features/jobs_database_status.feature
index 42b67a5..21441a8 100644
--- a/unity-test/features/jobs_database_status.feature
+++ b/unity-test/features/jobs_database_status.feature
@@ -13,36 +13,14 @@ Feature: The Jobs Database has Job Status
         | project_name | process_name     |
         | sounder_sips |  L1A             |
         | sounder_sips |  L1B             |
-        | sounder_sips |  chirp |
+        | sounder_sips |  chirp         |
 
     Scenario Outline: Following a job execution request of a deployed algorithm process that results in a success, the jobs database reflects the success
-        Given the <project_name> <process_name> algorithm process has been deployed to the ADES
+        Given the sounder_sips chirp algorithm process has been deployed to the ADES
         And a WPS-T request is made to execute the process
         And the HTTP response contains a status code of 201
         And the response includes a Location header
         And the Location header contains a job ID
         And the job runs successfully
         When the status of the job is queried through the jobs database
-        Then the jobs status is "succeeded"
-
-        Examples:
-        | project_name | process_name     |
-        | sounder_sips |  L1A             |
-        | sounder_sips |  L1B             |
-        | sounder_sips |  chirp |
-
-    Scenario Outline: Following a job execution request of a deployed algorithm process, the jobs database show that the job is running
-        Given the <project_name> <process_name> algorithm process has been deployed to the ADES
-        And a WPS-T request is made to execute the process
-        And the HTTP response contains a status code of 201
-        And the response includes a Location header
-        And the Location header contains a job ID
-        And the job is running
-        When the status of the job is queried through the jobs database
-        Then the job status is "running"
-
-        Examples:
-        | project_name | process_name     |
-        | sounder_sips |  L1A             |
-        | sounder_sips |  L1B             |
-        | sounder_sips |  chirp |
\ No newline at end of file
+        Then the job status is "succeeded"
\ No newline at end of file
diff --git a/unity-test/step_defs/test_job_database_status.py b/unity-test/step_defs/test_job_database_status.py
index 17baad7..31e025a 100644
--- a/unity-test/step_defs/test_job_database_status.py
+++ b/unity-test/step_defs/test_job_database_status.py
@@ -1,5 +1,6 @@
 from pytest_bdd import scenario, given, when, then, parsers
 from elasticsearch import Elasticsearch
+import pytest
 import backoff
 import requests
 from .conftest import FEATURES_DIR, _request_job_status_by_id
@@ -20,15 +21,16 @@ def test_following_processing_request_job_added_to_jobs_database():
     FEATURE_FILE,
     "Following a job execution request of a deployed algorithm process that results in a success, the jobs database reflects the success",
 )
-def test_following_successful_processing_request_job_added_to_job_database_status_is_success():
+def test_following_successful_processing_request_job_added_to_job_database_status_is_succeeded():
     pass
 
-@scenario(
-    FEATURE_FILE,
-    "Following a job execution request of a deployed algorithm process, the jobs database show that the job is running",
-)
-def test_following_processing_request_job_added_to_jobs_database():
-    pass
+# @scenario(
+#     FEATURE_FILE,
+#     "Following a job execution request of a deployed algorithm process, the jobs database show that the job is running",
+# )
+# @pytest.mark.skip(reason="Non-deterministic timing on sounder_sips processes make this test fail often")
+# def test_following_processing_request_job_gets_running_status():
+#     pass
 
 def fatal_status(e):
     fatal = False

From dd63be6bae22c6414c7ce4d602ad95d422d64588 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <124934265+ryanhunter-jpl@users.noreply.github.com>
Date: Tue, 22 Aug 2023 17:00:24 -0700
Subject: [PATCH 11/57] Update execution_post_request_body.json for chirp

---
 .../sounder_sips/chirp/execution_post_request_body.json   | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/unity-test/data/sounder_sips/chirp/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
index af155e0..67846df 100644
--- a/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
+++ b/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
@@ -2,10 +2,6 @@
   "mode": "async",
   "response": "document",
   "inputs": [
-    {
-      "id": "job_id",
-      "data": "abc123"
-    },
     {
       "id": "input_processing_labels",
       "data": ["label1", "label2"]
@@ -16,11 +12,11 @@
     },
     {
       "id": "input_cmr_search_start_time",
-      "data": "2016-08-22"
+      "data": "2016-08-22T00:10:00Z"
     },
     {
       "id": "input_cmr_search_stop_time",
-      "data": "2016-09-06"
+      "data": "2016-08-23T00:11:00Z"
     },
     {
       "id": "input_cmr_edl_user",

From 32c2082a699c6eba9876311c353840349fefc7fa Mon Sep 17 00:00:00 2001
From: Ryan Hunter <124934265+ryanhunter-jpl@users.noreply.github.com>
Date: Tue, 22 Aug 2023 17:00:53 -0700
Subject: [PATCH 12/57] Update deploy_post_request_body.json for chirp

---
 .../sounder_sips/chirp/deploy_post_request_body.json   | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/unity-test/data/sounder_sips/chirp/deploy_post_request_body.json b/unity-test/data/sounder_sips/chirp/deploy_post_request_body.json
index df2ede7..bac7919 100644
--- a/unity-test/data/sounder_sips/chirp/deploy_post_request_body.json
+++ b/unity-test/data/sounder_sips/chirp/deploy_post_request_body.json
@@ -14,16 +14,6 @@
       "abstract": "chirp",
       "keywords": [],
       "inputs": [
-        {
-          "id": "job_id",
-          "title": "job_id",
-          "formats": [
-            {
-              "mimeType": "text",
-              "default": true
-            }
-          ]
-        },
         {
           "id": "input_processing_labels",
           "title": "input_processing_labels",

From 84d1d72969459071b9120001d3ceafa86ae938db Mon Sep 17 00:00:00 2001
From: Rishi Verma <riverma@users.noreply.github.com>
Date: Wed, 23 Aug 2023 12:02:03 -0700
Subject: [PATCH 13/57] Redirecting to parent U-SPS repository

---
 CHANGELOG.md | 16 +------------
 README.md    | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ecf4858..23ece7c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,15 +1 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.0] - 2022-04-20
-
-### Added 
-
-- First implementation of HySDS services as Docker containers
-- Shell scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using the kubectl client
-- Terraform scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using Terraform
-
+Please see the parent repository's [CHANGELOG.md](https://github.com/unity-sds/unity-sps/blob/main/CHANGELOG.md) for a history of changes.
\ No newline at end of file
diff --git a/README.md b/README.md
index 24dbf02..50ec5fc 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,63 @@
-# unity-sps-prototype
\ No newline at end of file
+<!-- Header block for project -->
+<hr>
+
+<div align="center">
+
+<!-- ☝️ Replace with your logo (if applicable) via ![](https://uri-to-your-logo-image) ☝️ -->
+<!-- ☝️ If you see logo rendering errors, make sure you're not using indentation, or try an HTML IMG tag -->
+
+<h1 align="center">Unity SPS Prototype</h1>
+<!-- ☝️ Replace with your repo name ☝️ -->
+
+</div>
+
+<pre align="center">This is a prototype repository related to Unity SPS.</pre>
+<!-- ☝️ Replace with a single sentence describing the purpose of your repo / proj ☝️ -->
+
+<!-- Header block for project -->
+
+<!-- ☝️ Add badges via: https://shields.io e.g. ![](https://img.shields.io/github/your_chosen_action/your_org/your_repo) ☝️ -->
+
+<!-- ☝️ Screenshot of your software (if applicable) via ![](https://uri-to-your-screenshot) ☝️ -->
+
+For detailed information about this repository and its purpose, please refer to the parent repository, Unity SPS, at [https://github.com/unity-sds/unity-sps](https://github.com/unity-sds/unity-sps).
+
+## Features
+
+* Please consult the parent repository [unity-sps](https://github.com/unity-sds/unity-sps) for features related to all Unity SPS repositories.
+
+<!-- ☝️ Replace with a bullet-point list of your features ☝️ -->
+
+## Contents
+
+* [Quick Start](#quick-start)
+* [Changelog](#changelog)
+* [FAQ](#frequently-asked-questions-faq)
+* [Contributing Guide](#contributing)
+* [License](#license)
+* [Support](#support)
+
+## Quick Start
+
+For comprehensive details, guidelines, and documentation, refer to the parent repository: [unity-sps](https://github.com/unity-sds/unity-sps).
+
+
+## Changelog
+
+Please see the parent repository's [CHANGELOG.md](https://github.com/unity-sds/unity-sps/blob/main/CHANGELOG.md) for a history of changes.
+
+## Frequently Asked Questions (FAQ)
+
+For any questions, consult the FAQ of the parent repository [unity-sps](https://github.com/unity-sds/unity-sps).
+
+## Contributing
+
+To contribute, kindly refer to the guidelines in the parent repository: [unity-sps](https://github.com/unity-sds/unity-sps).
+
+## License
+
+See our: [LICENSE](LICENSE)
+
+## Support
+
+For support, please contact the maintainers of the parent repository [unity-sps](https://github.com/unity-sds/unity-sps).

From 352773101eaffede29342bbfae9553981d1f76e4 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <124934265+ryanhunter-jpl@users.noreply.github.com>
Date: Mon, 28 Aug 2023 14:13:12 -0700
Subject: [PATCH 14/57] Update regression_test_mcp_dev.yml to use right default
 variable

---
 .github/workflows/regression_test_mcp_dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/regression_test_mcp_dev.yml b/.github/workflows/regression_test_mcp_dev.yml
index 8ff3bf9..ea6a77a 100644
--- a/.github/workflows/regression_test_mcp_dev.yml
+++ b/.github/workflows/regression_test_mcp_dev.yml
@@ -50,6 +50,6 @@ jobs:
           pytest -s
           --process-service-endpoint=${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_DEV_PROCESS_SERVICE_ENDPOINT }}
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_DEV_SPS_API_SERVICE_ENDPOINT }}
-          --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}
+          --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_DEV_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
           -rs

From beb21c2c02a396d5b7d6b99f39b0f67b61887c9f Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 29 Aug 2023 16:37:43 -0700
Subject: [PATCH 15/57] add jobs db url to wpst env variables and ssm

---
 .../terraform-unity-sps-hysds-cluster/ades_wpst.tf         | 6 +++++-
 .../terraform-unity-sps-hysds-cluster/jobs_db.tf           | 7 +++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 7562c57..932b803 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -242,6 +242,10 @@ resource "kubernetes_deployment" "ades-wpst-api" {
             name  = "JOBS_DATA_SNS_TOPIC_ARN"
             value = aws_sns_topic.jobs_data.arn
           }
+          env {
+            name = "JOBS_DB_URL"
+            value = "http://${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}"
+          }
           port {
             container_port = 5000
           }
@@ -282,4 +286,4 @@ resource "kubernetes_deployment" "ades-wpst-api" {
     data.kubernetes_service.mozart-es,
     data.kubernetes_service.grq-es
   ]
-}
+}
\ No newline at end of file
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/jobs_db.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
index 4a4aca5..0d5828e 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
@@ -273,6 +273,13 @@ resource "aws_lambda_event_source_mapping" "sqs_event_source_mapping_jobs_data_i
   function_name    = aws_lambda_function.jobs_data_ingest.function_name
 }
 
+resource "aws_ssm_parameter" "jobs-db-url-param" {
+  name        = "/unity/sps/${var.deployment_name}/jobsDb/url"
+  description = "Full URL of the jobs db load balancer, including port for accesing jobs db"
+  type        = "String"
+  value       = "http://${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}"
+}
+
 # resource "aws_elasticsearch_domain" "jobs_database" {
 #   domain_name           = "${var.project}-${var.venue}-${var.service_area}-es-jobs-${local.counter}"
 #   elasticsearch_version = "7.10"

From ccc14a8a88f6b8c3b4e20e853037f809e0af7e30 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 29 Aug 2023 17:04:54 -0700
Subject: [PATCH 16/57] dates in chirp test from 1 day and 1 hour to 1 hour

---
 .../data/sounder_sips/chirp/execution_post_request_body.json    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unity-test/data/sounder_sips/chirp/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
index 67846df..54743c6 100644
--- a/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
+++ b/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
@@ -16,7 +16,7 @@
     },
     {
       "id": "input_cmr_search_stop_time",
-      "data": "2016-08-23T00:11:00Z"
+      "data": "2016-08-22T00:11:00Z"
     },
     {
       "id": "input_cmr_edl_user",

From f6a3a758f6e96cba60ba48eabd9de9af3e762767 Mon Sep 17 00:00:00 2001
From: Rishi Verma <riverma@users.noreply.github.com>
Date: Mon, 11 Sep 2023 12:13:57 -0700
Subject: [PATCH 17/57] Update README.md

Added missing info in README to ensure full compliance with leaderboard best practices. See: https://github.com/unity-sds/unity-project-management/blob/main/standards-compliance/leaderboard.md
---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 50ec5fc..ffba1fe 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
 
 <div align="center">
 
-<!-- ☝️ Replace with your logo (if applicable) via ![](https://uri-to-your-logo-image) ☝️ -->
+![logo](https://user-images.githubusercontent.com/3129134/163255685-857aa780-880f-4c09-b08c-4b53bf4af54d.png)
 <!-- ☝️ If you see logo rendering errors, make sure you're not using indentation, or try an HTML IMG tag -->
 
 <h1 align="center">Unity SPS Prototype</h1>
@@ -20,6 +20,8 @@
 
 <!-- ☝️ Screenshot of your software (if applicable) via ![](https://uri-to-your-screenshot) ☝️ -->
 
+[Unity Docs](https://unity-sds.gitbook.io/docs/) | [Unity-SPS Docs](https://unity-sds.gitbook.io/docs/developer-docs/science-processing) | [Issue Tracker](https://github.com/unity-sds/unity-sps-prototype/issues)
+
 For detailed information about this repository and its purpose, please refer to the parent repository, Unity SPS, at [https://github.com/unity-sds/unity-sps](https://github.com/unity-sds/unity-sps).
 
 ## Features

From dbe5f4f7132294d20a15ebcd0da722517b5a77ea Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 11 Sep 2023 13:07:51 -0700
Subject: [PATCH 18/57] Add tags to remaining untagged resources

---
 CHANGELOG.md                                  | 15 ------------
 .../eks_node_groups.tf                        | 24 +++++++++++--------
 .../variables.tf                              |  5 ----
 terraform-unity/README.md                     |  3 +--
 terraform-unity/main.tf                       |  1 -
 terraform-unity/variables.tf                  |  5 ----
 6 files changed, 15 insertions(+), 38 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c57c47..1d5de83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,16 +1 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.0] - 2022-04-20
-
-### Added
-
-- First implementation of HySDS services as Docker containers
-- Shell scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using the kubectl client
-- Terraform scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using Terraform
-
 Please see the parent repository's [CHANGELOG.md](https://github.com/unity-sds/unity-sps/blob/main/CHANGELOG.md) for a history of changes.
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
index 0a00656..b017d48 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
@@ -19,6 +19,10 @@ locals {
 #   node_group_name = "defaultgroupNodeGroup"
 # }
 
+data "aws_ssm_parameter" "mcp_linux_eks_optimized_ami" {
+  name = "/unity/account/ami/eksClusterAmi"
+}
+
 data "aws_launch_template" "default_group_node_group" {
   name = var.default_group_node_group_launch_template_name
 }
@@ -75,7 +79,7 @@ data "aws_security_groups" "sps-cluster-sg" {
 resource "aws_launch_template" "verdi_node_group_launch_template" {
   name = "${var.project}-${var.venue}-${var.service_area}-${var.deployment_name}-EC2-VerdiNodeGroupLaunchTemplate"
 
-  image_id = var.mcp_linux_eks_optimized_ami
+  image_id = data.aws_ssm_parameter.mcp_linux_eks_optimized_ami.value
 
   block_device_mappings {
     device_name = "/dev/xvda"
@@ -134,10 +138,10 @@ resource "aws_eks_node_group" "verdi" {
     max_size     = var.verdi_node_group_scaling_config.max_size
   }
   launch_template {
-    id      = data.aws_launch_template.default_group_node_group.id
-    version = data.aws_launch_template.default_group_node_group.latest_version
-    # id      = aws_launch_template.verdi_node_group_launch_template.id
-    # version = aws_launch_template.verdi_node_group_launch_template.latest_version
+    # id      = data.aws_launch_template.default_group_node_group.id
+    # version = data.aws_launch_template.default_group_node_group.latest_version
+    id      = aws_launch_template.verdi_node_group_launch_template.id
+    version = aws_launch_template.verdi_node_group_launch_template.latest_version
   }
   tags = merge(local.common_tags, {
     # Add or overwrite specific tags for this resource
@@ -226,7 +230,7 @@ resource "aws_iam_role_policy_attachment" "eks_sps_api_node_group_scaling_policy
 resource "aws_launch_template" "sps_api_node_group_launch_template" {
   name = "${var.project}-${var.venue}-${var.service_area}-${var.deployment_name}-EC2-SPSPAPINodeGroupLaunchTemplate"
 
-  image_id = var.mcp_linux_eks_optimized_ami
+  image_id = data.aws_ssm_parameter.mcp_linux_eks_optimized_ami.value
 
   block_device_mappings {
     device_name = "/dev/xvda"
@@ -285,10 +289,10 @@ resource "aws_eks_node_group" "sps_api" {
     max_size     = 1
   }
   launch_template {
-    id      = data.aws_launch_template.default_group_node_group.id
-    version = data.aws_launch_template.default_group_node_group.latest_version
-    # id      = aws_launch_template.sps_api_node_group_launch_template.id
-    # version = aws_launch_template.sps_api_node_group_launch_template.latest_version
+    # id      = data.aws_launch_template.default_group_node_group.id
+    # version = data.aws_launch_template.default_group_node_group.latest_version
+    id      = aws_launch_template.sps_api_node_group_launch_template.id
+    version = aws_launch_template.sps_api_node_group_launch_template.latest_version
   }
   tags = merge(local.common_tags, {
     # Add or overwrite specific tags for this resource
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
index d49699a..113d226 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -195,11 +195,6 @@ variable "verdi_node_group_instance_types" {
   default     = ["m3.medium"]
 }
 
-variable "mcp_linux_eks_optimized_ami" {
-  description = "value"
-  type        = string
-  default     = "ami-04db7a1ae7708642e"
-}
 variable "add_routes_to_api_gateway" {
   description = "If true, adds routes to api gateway configured in account"
   type        = bool
diff --git a/terraform-unity/README.md b/terraform-unity/README.md
index 7b7b3f8..4b8bea4 100644
--- a/terraform-unity/README.md
+++ b/terraform-unity/README.md
@@ -211,11 +211,10 @@ $ rm terraform.tf.backup
 | <a name="input_default_group_node_group_launch_template_name"></a> [default\_group\_node\_group\_launch\_template\_name](#input\_default\_group\_node\_group\_launch\_template\_name) | value | `string` | `null` | no |
 | <a name="input_default_group_node_group_name"></a> [default\_group\_node\_group\_name](#input\_default\_group\_node\_group\_name) | value | `string` | `null` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Unique name of this deployment in the account. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.1.0"<br>}</pre> | no |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"<br>}</pre> | no |
 | <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | The name of the EKS cluster. | `string` | n/a | yes |
 | <a name="input_elb_subnets"></a> [elb\_subnets](#input\_elb\_subnets) | value | `string` | `null` | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes |
-| <a name="input_mcp_linux_eks_optimized_ami"></a> [mcp\_linux\_eks\_optimized\_ami](#input\_mcp\_linux\_eks\_optimized\_ami) | value | `string` | `"ami-04db7a1ae7708642e"` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace for the Unity SPS HySDS-related Kubernetes resources | `string` | `"unity-sps"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `null` | no |
 | <a name="input_region"></a> [region](#input\_region) | The AWS region | `string` | `"us-west-2"` | no |
diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 4a622be..3f381ee 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -35,6 +35,5 @@ module "unity-sps-hysds-cluster" {
   verdi_node_group_capacity_type                = var.verdi_node_group_capacity_type
   verdi_node_group_scaling_config               = var.verdi_node_group_scaling_config
   verdi_node_group_instance_types               = var.verdi_node_group_instance_types
-  mcp_linux_eks_optimized_ami                   = var.mcp_linux_eks_optimized_ami
   add_routes_to_api_gateway                     = var.add_routes_to_api_gateway
 }
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index fd70fdf..f2ed333 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -176,11 +176,6 @@ variable "verdi_node_group_instance_types" {
   default     = ["m3.medium"]
 }
 
-variable "mcp_linux_eks_optimized_ami" {
-  description = "value"
-  type        = string
-  default     = "ami-04db7a1ae7708642e"
-}
 variable "add_routes_to_api_gateway" {
   description = "If true, adds routes to api gateway configured in account"
   type        = bool

From ee7088b9ac01d655d73189201a17484567e24d40 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 11 Sep 2023 13:15:26 -0700
Subject: [PATCH 19/57] Format terraform

---
 .../terraform-unity-sps-hysds-cluster/ades_wpst.tf              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 1955648..04798b3 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -248,7 +248,7 @@ resource "kubernetes_deployment" "ades-wpst-api" {
             value = aws_sns_topic.jobs_data.arn
           }
           env {
-            name = "JOBS_DB_URL"
+            name  = "JOBS_DB_URL"
             value = "http://${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}"
           }
           port {

From 36950fb276f9cb0d6d38f786c8edea26af0d4497 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 11 Sep 2023 13:39:10 -0700
Subject: [PATCH 20/57] Format and lint terraform

---
 .../terraform-unity-sps-hysds-cluster/README.md           | 8 +++-----
 .../terraform-unity-sps-hysds-cluster/eks_node_groups.tf  | 8 --------
 .../terraform-unity-sps-hysds-cluster/variables.tf        | 5 -----
 terraform-unity/README.md                                 | 2 --
 terraform-unity/main.tf                                   | 1 -
 terraform-unity/ssm_params.tf                             | 5 -----
 terraform-unity/variables.tf                              | 5 -----
 7 files changed, 3 insertions(+), 31 deletions(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/README.md b/terraform-modules/terraform-unity-sps-hysds-cluster/README.md
index cb8f1a5..69f89fc 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/README.md
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/README.md
@@ -67,8 +67,8 @@ No modules.
 | [aws_sns_topic_subscription.sqs_subscription](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sns_topic_subscription) | resource |
 | [aws_sqs_queue.jobs_data_ingest_queue](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sqs_queue) | resource |
 | [aws_sqs_queue_policy.jobs_data_queue](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/sqs_queue_policy) | resource |
+| [aws_ssm_parameter.jobs-db-url-param](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/ssm_parameter) | resource |
 | [aws_ssm_parameter.sps-api-hostname-param](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/ssm_parameter) | resource |
-| [aws_ssm_parameter.sps-api-port-param](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/resources/ssm_parameter) | resource |
 | [helm_release.grq2-es](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
 | [helm_release.jobs-es](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
 | [helm_release.mozart-es](https://registry.terraform.io/providers/hashicorp/helm/2.6.0/docs/resources/release) | resource |
@@ -126,11 +126,11 @@ No modules.
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/caller_identity) | data source |
 | [aws_efs_mount_target.uads-development-efs-fsmt](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/efs_mount_target) | data source |
 | [aws_eks_cluster.sps-cluster](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/eks_cluster) | data source |
-| [aws_launch_template.default_group_node_group](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/launch_template) | data source |
 | [aws_security_groups.sps-cluster-sg](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/security_groups) | data source |
 | [aws_ssm_parameter.api_gateway_rest_api_id](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.api_gateway_rest_api_lambda_authorizer_id](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.api_gateway_rest_api_root_resource_id](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
+| [aws_ssm_parameter.mcp_linux_eks_optimized_ami](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_subnets.eks_subnets](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/subnets) | data source |
 | [aws_vpc.eks_vpc](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/vpc) | data source |
 | [kubernetes_service.grq-es](https://registry.terraform.io/providers/hashicorp/kubernetes/2.19.0/docs/data-sources/service) | data source |
@@ -149,14 +149,12 @@ No modules.
 | <a name="input_container_registry_username"></a> [container\_registry\_username](#input\_container\_registry\_username) | value | `string` | `"drewm-jpl"` | no |
 | <a name="input_counter"></a> [counter](#input\_counter) | value | `string` | `""` | no |
 | <a name="input_datasets_filename"></a> [datasets\_filename](#input\_datasets\_filename) | value | `string` | `"datasets.remote.template.json"` | no |
-| <a name="input_default_group_node_group_launch_template_name"></a> [default\_group\_node\_group\_launch\_template\_name](#input\_default\_group\_node\_group\_launch\_template\_name) | value | `string` | n/a | yes |
 | <a name="input_default_group_node_group_name"></a> [default\_group\_node\_group\_name](#input\_default\_group\_node\_group\_name) | value | `string` | `"defaultgroupNodeGroup"` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Unique name of this deployment in the account. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.1.0"<br>}</pre> | no |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"<br>}</pre> | no |
 | <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | value | `string` | n/a | yes |
 | <a name="input_elb_subnets"></a> [elb\_subnets](#input\_elb\_subnets) | value | `string` | n/a | yes |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes |
-| <a name="input_mcp_linux_eks_optimized_ami"></a> [mcp\_linux\_eks\_optimized\_ami](#input\_mcp\_linux\_eks\_optimized\_ami) | value | `string` | `"ami-04db7a1ae7708642e"` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | Namespace for the Unity SPS HySDS-related Kubernetes resources | `string` | `"unity-sps"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no |
 | <a name="input_region"></a> [region](#input\_region) | The AWS region | `string` | `"us-west-2"` | no |
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
index b017d48..d354cba 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
@@ -23,10 +23,6 @@ data "aws_ssm_parameter" "mcp_linux_eks_optimized_ami" {
   name = "/unity/account/ami/eksClusterAmi"
 }
 
-data "aws_launch_template" "default_group_node_group" {
-  name = var.default_group_node_group_launch_template_name
-}
-
 resource "aws_iam_role" "eks_verdi_node_role" {
   name = "${var.project}-${var.venue}-${var.service_area}-EKS-VerdiNodeGroupIAMRole-${local.counter}"
   assume_role_policy = jsonencode({
@@ -138,8 +134,6 @@ resource "aws_eks_node_group" "verdi" {
     max_size     = var.verdi_node_group_scaling_config.max_size
   }
   launch_template {
-    # id      = data.aws_launch_template.default_group_node_group.id
-    # version = data.aws_launch_template.default_group_node_group.latest_version
     id      = aws_launch_template.verdi_node_group_launch_template.id
     version = aws_launch_template.verdi_node_group_launch_template.latest_version
   }
@@ -289,8 +283,6 @@ resource "aws_eks_node_group" "sps_api" {
     max_size     = 1
   }
   launch_template {
-    # id      = data.aws_launch_template.default_group_node_group.id
-    # version = data.aws_launch_template.default_group_node_group.latest_version
     id      = aws_launch_template.sps_api_node_group_launch_template.id
     version = aws_launch_template.sps_api_node_group_launch_template.latest_version
   }
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
index 113d226..e8f5eff 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -168,11 +168,6 @@ variable "default_group_node_group_name" {
   default     = "defaultgroupNodeGroup"
 }
 
-variable "default_group_node_group_launch_template_name" {
-  description = "value"
-  type        = string
-}
-
 variable "verdi_node_group_capacity_type" {
   description = "value"
   type        = string
diff --git a/terraform-unity/README.md b/terraform-unity/README.md
index 4b8bea4..98a96d9 100644
--- a/terraform-unity/README.md
+++ b/terraform-unity/README.md
@@ -188,7 +188,6 @@ $ rm terraform.tf.backup
 |------|------|
 | [aws_ssm_parameter.account_project](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.account_venue](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
-| [aws_ssm_parameter.default_group_node_group_launch_template_name](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.default_node_group_name](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.eks_private_subnets](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.ghcr_pat](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
@@ -208,7 +207,6 @@ $ rm terraform.tf.backup
 | <a name="input_container_registry_username"></a> [container\_registry\_username](#input\_container\_registry\_username) | value | `string` | `"drewm-jpl"` | no |
 | <a name="input_counter"></a> [counter](#input\_counter) | value | `string` | `""` | no |
 | <a name="input_datasets_filename"></a> [datasets\_filename](#input\_datasets\_filename) | value | `string` | `"datasets.remote.template.json"` | no |
-| <a name="input_default_group_node_group_launch_template_name"></a> [default\_group\_node\_group\_launch\_template\_name](#input\_default\_group\_node\_group\_launch\_template\_name) | value | `string` | `null` | no |
 | <a name="input_default_group_node_group_name"></a> [default\_group\_node\_group\_name](#input\_default\_group\_node\_group\_name) | value | `string` | `null` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Unique name of this deployment in the account. | `string` | n/a | yes |
 | <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"<br>}</pre> | no |
diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 3f381ee..54897ae 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -30,7 +30,6 @@ module "unity-sps-hysds-cluster" {
   eks_cluster_name                              = var.eks_cluster_name
   elb_subnets                                   = var.elb_subnets == null ? data.aws_ssm_parameter.eks_private_subnets[0].value : var.elb_subnets
   default_group_node_group_name                 = var.default_group_node_group_name == null ? data.aws_ssm_parameter.default_node_group_name[0].value : var.default_group_node_group_name
-  default_group_node_group_launch_template_name = var.default_group_node_group_launch_template_name == null ? data.aws_ssm_parameter.default_group_node_group_launch_template_name[0].value : var.default_group_node_group_launch_template_name
   deployment_name                               = var.deployment_name
   verdi_node_group_capacity_type                = var.verdi_node_group_capacity_type
   verdi_node_group_scaling_config               = var.verdi_node_group_scaling_config
diff --git a/terraform-unity/ssm_params.tf b/terraform-unity/ssm_params.tf
index b49449f..057dd00 100644
--- a/terraform-unity/ssm_params.tf
+++ b/terraform-unity/ssm_params.tf
@@ -11,11 +11,6 @@ data "aws_ssm_parameter" "uds_dapa_api" {
   name = format("/%s-%s-%s-deployment-uds_dapa_api", var.project == null ? data.aws_ssm_parameter.account_project[0].value : var.project, var.venue == null ? data.aws_ssm_parameter.account_venue[0].value : var.venue, var.service_area)
 }
 
-data "aws_ssm_parameter" "default_group_node_group_launch_template_name" {
-  count = var.default_group_node_group_launch_template_name == null ? 1 : 0
-  name  = "/unity/extensions/eks/${var.eks_cluster_name}/nodeGroups/default/launchTemplateName"
-}
-
 data "aws_ssm_parameter" "default_node_group_name" {
   count = var.default_group_node_group_name == null ? 1 : 0
   name  = "/unity/extensions/eks/${var.eks_cluster_name}/nodeGroups/default/name"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index f2ed333..cff6536 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -142,11 +142,6 @@ variable "elb_subnets" {
   type        = string
   default     = null
 }
-variable "default_group_node_group_launch_template_name" {
-  description = "value"
-  type        = string
-  default     = null
-}
 
 variable "default_group_node_group_name" {
   description = "value"

From d6fd22aba48af00d72ca14a8cb47dab519b72ee6 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 11 Sep 2023 13:40:26 -0700
Subject: [PATCH 21/57] Format and lint terraform

---
 terraform-unity/main.tf | 58 ++++++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 54897ae..d4fd3e8 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -6,33 +6,33 @@
 # https://www.terraform.io/language/modules/syntax
 
 module "unity-sps-hysds-cluster" {
-  source                                        = "../terraform-modules/terraform-unity-sps-hysds-cluster"
-  release                                       = var.release
-  project                                       = var.project == null ? data.aws_ssm_parameter.account_project[0].value : var.project
-  namespace                                     = var.namespace
-  venue                                         = var.venue == null ? data.aws_ssm_parameter.account_venue[0].value : var.venue
-  region                                        = var.region
-  counter                                       = var.counter
-  kubeconfig_filepath                           = var.kubeconfig_filepath
-  docker_images                                 = var.docker_images
-  service_type                                  = var.service_type
-  service_port_map                              = var.service_port_map
-  celeryconfig_filename                         = var.celeryconfig_filename
-  datasets_filename                             = var.datasets_filename
-  container_registry_server                     = var.container_registry_server
-  container_registry_username                   = var.container_registry_username
-  container_registry_pat                        = data.aws_ssm_parameter.ghcr_pat.value
-  container_registry_owner                      = var.container_registry_owner
-  uds_staging_bucket                            = data.aws_ssm_parameter.uds_staging_bucket.value
-  uds_client_id                                 = data.aws_ssm_parameter.uds_client_id.value
-  uds_dapa_api                                  = data.aws_ssm_parameter.uds_dapa_api.value
-  uads_development_efs_fsmt_id                  = var.uads_development_efs_fsmt_id == null ? data.aws_ssm_parameter.uads_development_efs_fsmt_id[0].value : var.uads_development_efs_fsmt_id
-  eks_cluster_name                              = var.eks_cluster_name
-  elb_subnets                                   = var.elb_subnets == null ? data.aws_ssm_parameter.eks_private_subnets[0].value : var.elb_subnets
-  default_group_node_group_name                 = var.default_group_node_group_name == null ? data.aws_ssm_parameter.default_node_group_name[0].value : var.default_group_node_group_name
-  deployment_name                               = var.deployment_name
-  verdi_node_group_capacity_type                = var.verdi_node_group_capacity_type
-  verdi_node_group_scaling_config               = var.verdi_node_group_scaling_config
-  verdi_node_group_instance_types               = var.verdi_node_group_instance_types
-  add_routes_to_api_gateway                     = var.add_routes_to_api_gateway
+  source                          = "../terraform-modules/terraform-unity-sps-hysds-cluster"
+  release                         = var.release
+  project                         = var.project == null ? data.aws_ssm_parameter.account_project[0].value : var.project
+  namespace                       = var.namespace
+  venue                           = var.venue == null ? data.aws_ssm_parameter.account_venue[0].value : var.venue
+  region                          = var.region
+  counter                         = var.counter
+  kubeconfig_filepath             = var.kubeconfig_filepath
+  docker_images                   = var.docker_images
+  service_type                    = var.service_type
+  service_port_map                = var.service_port_map
+  celeryconfig_filename           = var.celeryconfig_filename
+  datasets_filename               = var.datasets_filename
+  container_registry_server       = var.container_registry_server
+  container_registry_username     = var.container_registry_username
+  container_registry_pat          = data.aws_ssm_parameter.ghcr_pat.value
+  container_registry_owner        = var.container_registry_owner
+  uds_staging_bucket              = data.aws_ssm_parameter.uds_staging_bucket.value
+  uds_client_id                   = data.aws_ssm_parameter.uds_client_id.value
+  uds_dapa_api                    = data.aws_ssm_parameter.uds_dapa_api.value
+  uads_development_efs_fsmt_id    = var.uads_development_efs_fsmt_id == null ? data.aws_ssm_parameter.uads_development_efs_fsmt_id[0].value : var.uads_development_efs_fsmt_id
+  eks_cluster_name                = var.eks_cluster_name
+  elb_subnets                     = var.elb_subnets == null ? data.aws_ssm_parameter.eks_private_subnets[0].value : var.elb_subnets
+  default_group_node_group_name   = var.default_group_node_group_name == null ? data.aws_ssm_parameter.default_node_group_name[0].value : var.default_group_node_group_name
+  deployment_name                 = var.deployment_name
+  verdi_node_group_capacity_type  = var.verdi_node_group_capacity_type
+  verdi_node_group_scaling_config = var.verdi_node_group_scaling_config
+  verdi_node_group_instance_types = var.verdi_node_group_instance_types
+  add_routes_to_api_gateway       = var.add_routes_to_api_gateway
 }

From f417a88440bf5db5288e1ba614b443f13e141269 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 11 Sep 2023 13:42:45 -0700
Subject: [PATCH 22/57] Format and lint terraform

---
 .../terraform-unity-sps-hysds-cluster/eks_node_groups.tf     | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
index d354cba..538d861 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
@@ -14,11 +14,6 @@ locals {
   ]
 }
 
-# data "aws_eks_node_group" "default_node_group" {
-#   cluster_name    = var.eks_cluster_name
-#   node_group_name = "defaultgroupNodeGroup"
-# }
-
 data "aws_ssm_parameter" "mcp_linux_eks_optimized_ami" {
   name = "/unity/account/ami/eksClusterAmi"
 }

From 31bf55685ddf7f9ac983b169bde1c9495efcc14c Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 11 Sep 2023 13:43:39 -0700
Subject: [PATCH 23/57] Format and lint terraform

---
 .../ades_wpst.tf                              | 45 +------------------
 1 file changed, 1 insertion(+), 44 deletions(-)

diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 04798b3..42fd1f8 100644
--- a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -41,49 +41,6 @@ resource "kubernetes_persistent_volume_claim" "ades-wpst-sqlite-pv-claim" {
   }
 }
 
-# resource "aws_security_group_rule" "allow_elb_ingress" {
-#   type      = "ingress"
-#   from_port = var.service_port_map.ades_wpst_api_service
-#   to_port   = var.service_port_map.ades_wpst_api_service
-#   protocol  = "tcp"
-#   # security_group_id        = "eks-cluster-sg-unity-dev-sps-hysds-eks-nightly-795451454"
-#   security_group_id        = "sg-0650d2b7e1e1ddeac"
-#   source_security_group_id = aws_elb.ades_wpst_api_elb.source_security_group_id
-# }
-
-# resource "aws_security_group_rule" "elb_ingress" {
-#   type      = "ingress"
-#   from_port = var.service_port_map.ades_wpst_api_service
-#   to_port   = var.service_port_map.ades_wpst_api_service
-#   protocol  = "tcp"
-#   # security_group_id = aws_elb.ades_wpst_api_elb.security_groups[0]
-#   security_group_id = aws_elb.ades_wpst_api_elb.source_security_group_id
-#   cidr_blocks       = ["0.0.0.0/0"]
-# }
-
-# resource "aws_elb" "ades_wpst_api_elb" {
-#   name    = "${var.project}-${var.venue}-${var.service_area}-ADES-ELB-${local.counter}"
-#   subnets = data.aws_eks_node_group.default_node_group.subnet_ids
-
-#   listener {
-#     instance_port     = 32000
-#     instance_protocol = "tcp"
-#     lb_port           = var.service_port_map.ades_wpst_api_service
-#     lb_protocol       = "tcp"
-#   }
-
-#   tags = merge(local.common_tags, {
-#     Name      = "${var.project}-${var.venue}-${var.service_area}-ADES-ELB-${local.counter}"
-#     Component = "ADES"
-#     Stack     = "ADES"
-#   })
-# }
-
-# resource "aws_autoscaling_attachment" "ades_wpst_api_elb_attachment" {
-#   autoscaling_group_name = data.aws_eks_node_group.default_node_group.resources[0].autoscaling_groups[0].name
-#   elb                    = aws_elb.ades_wpst_api_elb.id
-# }
-
 resource "kubernetes_service" "ades-wpst-api-service" {
   metadata {
     name      = "ades-wpst-api"
@@ -291,4 +248,4 @@ resource "kubernetes_deployment" "ades-wpst-api" {
     data.kubernetes_service.mozart-es,
     data.kubernetes_service.grq-es
   ]
-}
\ No newline at end of file
+}

From f51f9472a80e5fec47c17f9fbc64922cec015811 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 2 Oct 2023 08:38:29 -0700
Subject: [PATCH 24/57] venue specific job values in integration tests

---
 .../L1A/dev/execution_post_request_body.json  | 39 ++++++++++++++
 .../L1A/test/execution_post_request_body.json | 39 ++++++++++++++
 .../L1B/dev/execution_post_request_body.json  | 28 ++++++++++
 .../L1B/test/execution_post_request_body.json | 28 ++++++++++
 .../dev/execution_post_request_body.json      | 52 +++++++++++++++++++
 .../test/execution_post_request_body.json     | 52 +++++++++++++++++++
 unity-test/step_defs/conftest.py              | 16 +++++-
 unity-test/step_defs/utils.py                 | 11 ++--
 8 files changed, 259 insertions(+), 6 deletions(-)
 create mode 100644 unity-test/data/sounder_sips/L1A/dev/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/L1B/dev/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/chirp/dev/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json

diff --git a/unity-test/data/sounder_sips/L1A/dev/execution_post_request_body.json b/unity-test/data/sounder_sips/L1A/dev/execution_post_request_body.json
new file mode 100644
index 0000000..5e26d77
--- /dev/null
+++ b/unity-test/data/sounder_sips/L1A/dev/execution_post_request_body.json
@@ -0,0 +1,39 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_ephatt_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:DEV1:L0_SNPP_EphAtt___1"
+    },
+    {
+      "id": "input_science_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:DEV1:L0_SNPP_ATMS_SCIENCE___1"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:DEV1:SNDR_SNPP_ATMS_L1A_OUTPUT___1"
+    },
+    {
+      "id": "static_dir",
+      "data": {
+        "class": "Directory",
+        "path": "/tmp/SOUNDER_SIPS/STATIC_DATA"
+      }
+    },
+    {
+      "id": "start_datetime",
+      "data": "2016-01-14T08:00:00Z"
+    },
+    {
+      "id": "stop_datetime",
+      "data": "2016-01-14T11:59:59Z"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json b/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json
new file mode 100644
index 0000000..9a030ba
--- /dev/null
+++ b/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json
@@ -0,0 +1,39 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_ephatt_collection_id",
+      "data": "L0_SNPP_EphAtt___1"
+    },
+    {
+      "id": "input_science_collection_id",
+      "data": "L0_SNPP_ATMS_SCIENCE___1"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "SNDR_SNPP_ATMS_L1A_OUTPUT___1"
+    },
+    {
+      "id": "static_dir",
+      "data": {
+        "class": "Directory",
+        "path": "/tmp/SOUNDER_SIPS/STATIC_DATA"
+      }
+    },
+    {
+      "id": "start_datetime",
+      "data": "2016-01-14T08:00:00Z"
+    },
+    {
+      "id": "stop_datetime",
+      "data": "2016-01-14T11:59:59Z"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/L1B/dev/execution_post_request_body.json b/unity-test/data/sounder_sips/L1B/dev/execution_post_request_body.json
new file mode 100644
index 0000000..b57ecc6
--- /dev/null
+++ b/unity-test/data/sounder_sips/L1B/dev/execution_post_request_body.json
@@ -0,0 +1,28 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:DEV1:SNDR_SNPP_ATMS_L1A___1"
+    },
+    {
+      "id": "start_datetime",
+      "data": "2016-01-14T08:00:00Z"
+    },
+    {
+      "id": "stop_datetime",
+      "data": "2016-01-14T11:59:59Z"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:DEV1:SNDR_SNPP_ATMS_L1B_OUTPUT___1"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json b/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json
new file mode 100644
index 0000000..598d4b7
--- /dev/null
+++ b/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json
@@ -0,0 +1,28 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_collection_id",
+      "data": "SNDR_SNPP_ATMS_L1A___1"
+    },
+    {
+      "id": "start_datetime",
+      "data": "2016-01-14T08:00:00Z"
+    },
+    {
+      "id": "stop_datetime",
+      "data": "2016-01-14T11:59:59Z"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "SNDR_SNPP_ATMS_L1B_OUTPUT___1"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/chirp/dev/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/dev/execution_post_request_body.json
new file mode 100644
index 0000000..54743c6
--- /dev/null
+++ b/unity-test/data/sounder_sips/chirp/dev/execution_post_request_body.json
@@ -0,0 +1,52 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_processing_labels",
+      "data": ["label1", "label2"]
+    },
+    {
+      "id": "input_cmr_collection_name",
+      "data": "C2011289787-GES_DISC"
+    },
+    {
+      "id": "input_cmr_search_start_time",
+      "data": "2016-08-22T00:10:00Z"
+    },
+    {
+      "id": "input_cmr_search_stop_time",
+      "data": "2016-08-22T00:11:00Z"
+    },
+    {
+      "id": "input_cmr_edl_user",
+      "data": "cmr_user"
+    },
+    {
+      "id": "input_cmr_edl_pass",
+      "data": "cmr_pass"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "CHIRP_OUTPUT_COLLECTION"
+    },
+    {
+      "id": "output_data_bucket",
+      "data": "s3://unity-data-bucket"
+    },
+    {
+      "id": "input_daac_collection_shortname",
+      "data": "CHIRP_L1B"
+    },
+    {
+      "id": "input_daac_collection_sns",
+      "data": "arn:://SNS-arn"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json
new file mode 100644
index 0000000..54743c6
--- /dev/null
+++ b/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json
@@ -0,0 +1,52 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_processing_labels",
+      "data": ["label1", "label2"]
+    },
+    {
+      "id": "input_cmr_collection_name",
+      "data": "C2011289787-GES_DISC"
+    },
+    {
+      "id": "input_cmr_search_start_time",
+      "data": "2016-08-22T00:10:00Z"
+    },
+    {
+      "id": "input_cmr_search_stop_time",
+      "data": "2016-08-22T00:11:00Z"
+    },
+    {
+      "id": "input_cmr_edl_user",
+      "data": "cmr_user"
+    },
+    {
+      "id": "input_cmr_edl_pass",
+      "data": "cmr_pass"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "CHIRP_OUTPUT_COLLECTION"
+    },
+    {
+      "id": "output_data_bucket",
+      "data": "s3://unity-data-bucket"
+    },
+    {
+      "id": "input_daac_collection_shortname",
+      "data": "CHIRP_L1B"
+    },
+    {
+      "id": "input_daac_collection_sns",
+      "data": "arn:://SNS-arn"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/step_defs/conftest.py b/unity-test/step_defs/conftest.py
index e086a24..eddb1e4 100644
--- a/unity-test/step_defs/conftest.py
+++ b/unity-test/step_defs/conftest.py
@@ -36,6 +36,13 @@ def pytest_addoption(parser):
         help="The Sounder SIPS processes to test (L1A, L1B, chirp)",
         required=True,
     )
+    parser.addoption(
+        "--environment",
+        type=str,
+        action="store",
+        help="Environment to run test against. (dev, test)",
+        default=None
+    )
 
 @pytest.fixture(scope="module", autouse=True)
 def process_service_endpoint(request):
@@ -50,6 +57,10 @@ def sps_api_service_endpoint(request):
 def jobs_database_endpoint(request):
     return request.config.getoption("--jobs-database-endpoint")
 
+@pytest.fixture()
+def environment(request):
+    return request.config.getoption("--environment")
+
 @pytest.fixture()
 def jobs_database_client(jobs_database_endpoint):
     return Elasticsearch(jobs_database_endpoint)
@@ -60,7 +71,7 @@ def projects():
     return data
 
 @pytest.fixture
-def job_request_body(project_process_dict):
+def job_reques(project_process_dict):
     return reader.request_body(
         project_process_dict["project_name"],
         project_process_dict["process_name"],
@@ -218,11 +229,12 @@ def created_response(response):
     "a WPS-T request is made to execute the process",
     target_fixture="response",
 )
-def request_job_execution(process_service_endpoint, project_process_dict):
+def request_job_execution(process_service_endpoint, project_process_dict, environment):
     request_body = reader.request_body(
         project_process_dict["project_name"],
         project_process_dict["process_name"],
         reader.execution_post_request_body,
+        environment=environment
     )
     job_execution_response = _request_job_execution(
         process_service_endpoint, project_process_dict["process_name"], request_body
diff --git a/unity-test/step_defs/utils.py b/unity-test/step_defs/utils.py
index d7ab3de..fbed76f 100644
--- a/unity-test/step_defs/utils.py
+++ b/unity-test/step_defs/utils.py
@@ -10,10 +10,13 @@ def __init__(self):
         self.execution_post_request_body = "execution_post_request_body.json"
         self.start_prewarm_post_request_body = "start_prewarm_post_request_body.json"
 
-    def request_body(self, project_name, process_name, request_body_filename):
-        data_file_path = self.data_dir.joinpath(
-            project_name, process_name, request_body_filename
-        )
+    def request_body(self, project_name, process_name, request_body_filename, environment=None):
+        if environment:
+            data_file_path = [project_name, process_name, environment, request_body_filename]
+        else:
+            data_file_path = [project_name, process_name, request_body_filename]
+
+        data_file_path = self.data_dir.joinpath(*data_file_path)
 
         if not data_file_path.exists():
             raise ValueError(f"Data file not found: {data_file_path}")

From f28d8a57aade078b3b32556d52e2e17382b83f34 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 2 Oct 2023 08:52:06 -0700
Subject: [PATCH 25/57] add environment flag to workflows

---
 .github/workflows/regression_test_mcp_dev.yml  | 1 +
 .github/workflows/regression_test_mcp_test.yml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/regression_test_mcp_dev.yml b/.github/workflows/regression_test_mcp_dev.yml
index ea6a77a..43937f8 100644
--- a/.github/workflows/regression_test_mcp_dev.yml
+++ b/.github/workflows/regression_test_mcp_dev.yml
@@ -52,4 +52,5 @@ jobs:
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_DEV_SPS_API_SERVICE_ENDPOINT }}
           --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_DEV_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
+          --environment "dev"
           -rs
diff --git a/.github/workflows/regression_test_mcp_test.yml b/.github/workflows/regression_test_mcp_test.yml
index 34f83ca..f6d04f3 100644
--- a/.github/workflows/regression_test_mcp_test.yml
+++ b/.github/workflows/regression_test_mcp_test.yml
@@ -52,4 +52,5 @@ jobs:
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_TEST_SPS_API_SERVICE_ENDPOINT }}
           --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
+          --environment "test"
           -rs

From 42613370c52f4a4887f5fd0404cb1e40c93e4505 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 2 Oct 2023 10:04:05 -0700
Subject: [PATCH 26/57] fix job_request_body fixture for envs

---
 unity-test/step_defs/conftest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/unity-test/step_defs/conftest.py b/unity-test/step_defs/conftest.py
index eddb1e4..449ac3c 100644
--- a/unity-test/step_defs/conftest.py
+++ b/unity-test/step_defs/conftest.py
@@ -71,11 +71,12 @@ def projects():
     return data
 
 @pytest.fixture
-def job_reques(project_process_dict):
+def job_request_body(project_process_dict, environment):
     return reader.request_body(
         project_process_dict["project_name"],
         project_process_dict["process_name"],
         reader.execution_post_request_body,
+        environment=environment
     )
 
 @pytest.fixture

From 2a498453aba329511bd53b91d1dfa0b3f3a15878 Mon Sep 17 00:00:00 2001
From: Barber <barber@MT-306070.jpl.nasa.gov>
Date: Wed, 4 Oct 2023 08:34:52 +0100
Subject: [PATCH 27/57] move modules so they are loadable

---
 terraform-unity/main.tf                                         | 2 +-
 {terraform-modules => terraform-unity/modules}/README.md        | 0
 .../modules}/terraform-unity-secrets/.terraform.lock.hcl        | 0
 .../modules}/terraform-unity-secrets/README.md                  | 0
 .../modules}/terraform-unity-secrets/main.tf                    | 0
 .../modules}/terraform-unity-secrets/ssm_params.tf              | 0
 .../modules}/terraform-unity-secrets/variables.tf               | 0
 .../modules}/terraform-unity-secrets/versions.tf                | 0
 .../modules}/terraform-unity-sps-hysds-cluster/README.md        | 0
 .../modules}/terraform-unity-sps-hysds-cluster/ades_wpst.tf     | 0
 .../modules}/terraform-unity-sps-hysds-cluster/api_gateway.tf   | 0
 .../modules}/terraform-unity-sps-hysds-cluster/configs.tf       | 0
 .../modules}/terraform-unity-sps-hysds-cluster/efs.tf           | 0
 .../modules}/terraform-unity-sps-hysds-cluster/eks.tf           | 0
 .../terraform-unity-sps-hysds-cluster/eks_node_groups.tf        | 0
 .../terraform-unity-sps-hysds-cluster/eks_permissions.tf        | 0
 .../modules}/terraform-unity-sps-hysds-cluster/elastic.tf       | 0
 .../modules}/terraform-unity-sps-hysds-cluster/factotum.tf      | 0
 .../modules}/terraform-unity-sps-hysds-cluster/grq2.tf          | 0
 .../modules}/terraform-unity-sps-hysds-cluster/hysds_ui.tf      | 0
 .../modules}/terraform-unity-sps-hysds-cluster/jobs_db.tf       | 0
 .../modules}/terraform-unity-sps-hysds-cluster/logstash.tf      | 0
 .../modules}/terraform-unity-sps-hysds-cluster/main.tf          | 0
 .../modules}/terraform-unity-sps-hysds-cluster/mozart.tf        | 0
 .../modules}/terraform-unity-sps-hysds-cluster/orchestrator.tf  | 0
 .../modules}/terraform-unity-sps-hysds-cluster/outputs.tf       | 0
 .../modules}/terraform-unity-sps-hysds-cluster/rabbitmq.tf      | 0
 .../modules}/terraform-unity-sps-hysds-cluster/redis.tf         | 0
 .../modules}/terraform-unity-sps-hysds-cluster/sps_api.tf       | 0
 .../modules}/terraform-unity-sps-hysds-cluster/test.tf          | 0
 .../modules}/terraform-unity-sps-hysds-cluster/user-rules.tf    | 0
 .../modules}/terraform-unity-sps-hysds-cluster/variables.tf     | 0
 .../modules}/terraform-unity-sps-hysds-cluster/verdi.tf         | 0
 .../modules}/terraform-unity-sps-hysds-cluster/versions.tf      | 0
 34 files changed, 1 insertion(+), 1 deletion(-)
 rename {terraform-modules => terraform-unity/modules}/README.md (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-secrets/.terraform.lock.hcl (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-secrets/README.md (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-secrets/main.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-secrets/ssm_params.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-secrets/variables.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-secrets/versions.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/README.md (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/ades_wpst.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/api_gateway.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/configs.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/efs.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/eks.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/eks_node_groups.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/eks_permissions.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/elastic.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/factotum.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/grq2.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/hysds_ui.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/jobs_db.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/logstash.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/main.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/mozart.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/orchestrator.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/outputs.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/rabbitmq.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/redis.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/sps_api.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/test.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/user-rules.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/variables.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/verdi.tf (100%)
 rename {terraform-modules => terraform-unity/modules}/terraform-unity-sps-hysds-cluster/versions.tf (100%)

diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index d4fd3e8..2c691b0 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -6,7 +6,7 @@
 # https://www.terraform.io/language/modules/syntax
 
 module "unity-sps-hysds-cluster" {
-  source                          = "../terraform-modules/terraform-unity-sps-hysds-cluster"
+  source                          = "./modules/terraform-unity-sps-hysds-cluster"
   release                         = var.release
   project                         = var.project == null ? data.aws_ssm_parameter.account_project[0].value : var.project
   namespace                       = var.namespace
diff --git a/terraform-modules/README.md b/terraform-unity/modules/README.md
similarity index 100%
rename from terraform-modules/README.md
rename to terraform-unity/modules/README.md
diff --git a/terraform-modules/terraform-unity-secrets/.terraform.lock.hcl b/terraform-unity/modules/terraform-unity-secrets/.terraform.lock.hcl
similarity index 100%
rename from terraform-modules/terraform-unity-secrets/.terraform.lock.hcl
rename to terraform-unity/modules/terraform-unity-secrets/.terraform.lock.hcl
diff --git a/terraform-modules/terraform-unity-secrets/README.md b/terraform-unity/modules/terraform-unity-secrets/README.md
similarity index 100%
rename from terraform-modules/terraform-unity-secrets/README.md
rename to terraform-unity/modules/terraform-unity-secrets/README.md
diff --git a/terraform-modules/terraform-unity-secrets/main.tf b/terraform-unity/modules/terraform-unity-secrets/main.tf
similarity index 100%
rename from terraform-modules/terraform-unity-secrets/main.tf
rename to terraform-unity/modules/terraform-unity-secrets/main.tf
diff --git a/terraform-modules/terraform-unity-secrets/ssm_params.tf b/terraform-unity/modules/terraform-unity-secrets/ssm_params.tf
similarity index 100%
rename from terraform-modules/terraform-unity-secrets/ssm_params.tf
rename to terraform-unity/modules/terraform-unity-secrets/ssm_params.tf
diff --git a/terraform-modules/terraform-unity-secrets/variables.tf b/terraform-unity/modules/terraform-unity-secrets/variables.tf
similarity index 100%
rename from terraform-modules/terraform-unity-secrets/variables.tf
rename to terraform-unity/modules/terraform-unity-secrets/variables.tf
diff --git a/terraform-modules/terraform-unity-secrets/versions.tf b/terraform-unity/modules/terraform-unity-secrets/versions.tf
similarity index 100%
rename from terraform-modules/terraform-unity-secrets/versions.tf
rename to terraform-unity/modules/terraform-unity-secrets/versions.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/README.md b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/README.md
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/README.md
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/README.md
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/api_gateway.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/configs.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/configs.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/configs.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/configs.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/efs.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/efs.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/efs.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/efs.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/eks.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/eks.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/eks_permissions.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_permissions.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/eks_permissions.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_permissions.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/elastic.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/factotum.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/factotum.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/factotum.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/factotum.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/grq2.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/grq2.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/grq2.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/grq2.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/hysds_ui.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/hysds_ui.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/hysds_ui.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/hysds_ui.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/jobs_db.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/logstash.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/logstash.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/logstash.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/logstash.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/main.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/main.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/main.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/main.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/mozart.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/mozart.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/mozart.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/mozart.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/orchestrator.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/orchestrator.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/orchestrator.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/orchestrator.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/outputs.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/redis.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/redis.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/redis.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/redis.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/sps_api.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/test.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/test.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/test.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/test.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/user-rules.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/user-rules.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/user-rules.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/user-rules.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/variables.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/verdi.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/verdi.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/verdi.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/verdi.tf
diff --git a/terraform-modules/terraform-unity-sps-hysds-cluster/versions.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/versions.tf
similarity index 100%
rename from terraform-modules/terraform-unity-sps-hysds-cluster/versions.tf
rename to terraform-unity/modules/terraform-unity-sps-hysds-cluster/versions.tf

From d6360e3042248de4468566d74d02b913e20d6fbb Mon Sep 17 00:00:00 2001
From: Barber <barber@MT-306070.jpl.nasa.gov>
Date: Wed, 4 Oct 2023 08:42:41 +0100
Subject: [PATCH 28/57] pre init script to fetch kubeconfig

---
 terraform-unity/scripts/preinstall.sh | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100755 terraform-unity/scripts/preinstall.sh

diff --git a/terraform-unity/scripts/preinstall.sh b/terraform-unity/scripts/preinstall.sh
new file mode 100755
index 0000000..a339016
--- /dev/null
+++ b/terraform-unity/scripts/preinstall.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+mkdir $WORKDIR/k8s
+aws eks update-kubeconfig --name $EKS_NAME --kubeconfig $WORKDIR/k8s/kubernetes.yml

From 7bc78118f41478367c2c053b7efabafa0d7571e3 Mon Sep 17 00:00:00 2001
From: Barber <barber@MT-306070.jpl.nasa.gov>
Date: Wed, 4 Oct 2023 08:52:06 +0100
Subject: [PATCH 29/57] update var

---
 terraform-unity/scripts/preinstall.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-unity/scripts/preinstall.sh b/terraform-unity/scripts/preinstall.sh
index a339016..fdeacbe 100755
--- a/terraform-unity/scripts/preinstall.sh
+++ b/terraform-unity/scripts/preinstall.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 mkdir $WORKDIR/k8s
-aws eks update-kubeconfig --name $EKS_NAME --kubeconfig $WORKDIR/k8s/kubernetes.yml
+aws eks update-kubeconfig --name $UNITY_EKS --kubeconfig $WORKDIR/k8s/kubernetes.yml

From b5fc76d994d41557e3645cceb10051790775c9d8 Mon Sep 17 00:00:00 2001
From: Barber <barber@MT-306070.jpl.nasa.gov>
Date: Wed, 4 Oct 2023 09:01:31 +0100
Subject: [PATCH 30/57] terraform updates to make it work with internal
 loadbalancers, change paths for different module layout, additional variables
 for load balancer, update provider versions for compatability

---
 terraform-unity/main.tf                       |  1 +
 .../terraform-unity-secrets/versions.tf       |  2 +-
 .../ades_wpst.tf                              |  2 ++
 .../configs.tf                                | 30 +++++++++----------
 .../elastic.tf                                |  2 ++
 .../jobs_db.tf                                |  9 +++---
 .../sps_api.tf                                |  2 ++
 .../variables.tf                              | 12 ++++++++
 .../versions.tf                               |  2 +-
 terraform-unity/variables.tf                  |  7 +++++
 10 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 2c691b0..39ba645 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -35,4 +35,5 @@ module "unity-sps-hysds-cluster" {
   verdi_node_group_scaling_config = var.verdi_node_group_scaling_config
   verdi_node_group_instance_types = var.verdi_node_group_instance_types
   add_routes_to_api_gateway       = var.add_routes_to_api_gateway
+  tags                            = var.tags
 }
diff --git a/terraform-unity/modules/terraform-unity-secrets/versions.tf b/terraform-unity/modules/terraform-unity-secrets/versions.tf
index 4916554..2778f69 100644
--- a/terraform-unity/modules/terraform-unity-secrets/versions.tf
+++ b/terraform-unity/modules/terraform-unity-secrets/versions.tf
@@ -3,7 +3,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "4.57.1"
+      version = "~> 5.0"
     }
   }
 }
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 42fd1f8..2449ca8 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -53,6 +53,8 @@ resource "kubernetes_service" "ades-wpst-api-service" {
         "Stack"     = "adeswpst"
       }) : format("%s=%s", k, v)])
       "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
+      "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
+      "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
     }
   }
   spec {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/configs.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/configs.tf
index d5494ae..b8572cf 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/configs.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/configs.tf
@@ -4,7 +4,7 @@ resource "kubernetes_config_map" "mozart-settings" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "settings.cfg" = "${chomp(templatefile("${path.module}/../../hysds/mozart/rest_api/settings.cfg", {
+    "settings.cfg" = "${chomp(templatefile("${path.module}/../../../hysds/mozart/rest_api/settings.cfg", {
       rabbitmq_admin_port = var.service_port_map.rabbitmq_mgmt_service_cluster_rpc
       mozart_service_port = var.service_port_map.mozart_service
       mozart_es_port      = var.service_port_map.mozart_es
@@ -18,7 +18,7 @@ resource "kubernetes_config_map" "grq2-settings" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "settings.cfg" = "${chomp(templatefile("${path.module}/../../hysds/grq/rest_api/settings.cfg", {
+    "settings.cfg" = "${chomp(templatefile("${path.module}/../../../hysds/grq/rest_api/settings.cfg", {
       mozart_es_port     = var.service_port_map.mozart_es
       redis_service_port = var.service_port_map.redis_service
       grq2_es_port       = var.service_port_map.grq2_es
@@ -32,7 +32,7 @@ resource "kubernetes_config_map" "celeryconfig" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "celeryconfig.py" = "${chomp(templatefile("${path.module}/../../hysds/configs/${var.celeryconfig_filename}", {
+    "celeryconfig.py" = "${chomp(templatefile("${path.module}/../../../hysds/configs/${var.celeryconfig_filename}", {
       rabbitmq_service_port = var.service_port_map.rabbitmq_service_listener
       redis_service_port    = var.service_port_map.redis_service
       mozart_service_port   = var.service_port_map.mozart_service
@@ -49,7 +49,7 @@ resource "kubernetes_config_map" "netrc" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    ".netrc" = "${file("${path.module}/../../hysds/configs/.netrc")}"
+    ".netrc" = "${file("${path.module}/../../../hysds/configs/.netrc")}"
   }
 }
 
@@ -59,14 +59,14 @@ resource "kubernetes_config_map" "logstash-configs" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "job-status"    = "${file("${path.module}/../../hysds/mozart/logstash/job_status.template.json")}"
-    "event-status"  = "${file("${path.module}/../../hysds/mozart/logstash/event_status.template.json")}"
-    "worker-status" = "${file("${path.module}/../../hysds/mozart/logstash/worker_status.template.json")}"
-    "task-status"   = "${file("${path.module}/../../hysds/mozart/logstash/task_status.template.json")}"
-    "logstash-conf" = "${chomp(templatefile("${path.module}/../../hysds/mozart/logstash/logstash.conf", {
+    "job-status"    = "${file("${path.module}/../../../hysds/mozart/logstash/job_status.template.json")}"
+    "event-status"  = "${file("${path.module}/../../../hysds/mozart/logstash/event_status.template.json")}"
+    "worker-status" = "${file("${path.module}/../../../hysds/mozart/logstash/worker_status.template.json")}"
+    "task-status"   = "${file("${path.module}/../../../hysds/mozart/logstash/task_status.template.json")}"
+    "logstash-conf" = "${chomp(templatefile("${path.module}/../../../hysds/mozart/logstash/logstash.conf", {
       mozart_es_port = var.service_port_map.mozart_es
     }))}"
-    "logstash-yml" = "${chomp(templatefile("${path.module}/../../hysds/mozart/logstash/logstash.yml", {
+    "logstash-yml" = "${chomp(templatefile("${path.module}/../../../hysds/mozart/logstash/logstash.yml", {
       mozart_es_port = var.service_port_map.mozart_es
     }))}"
   }
@@ -142,7 +142,7 @@ resource "kubernetes_config_map" "supervisord-orchestrator" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "supervisord.conf" = "${file("${path.module}/../../hysds/orchestrator/supervisord.conf")}"
+    "supervisord.conf" = "${file("${path.module}/../../../hysds/orchestrator/supervisord.conf")}"
   }
 }
 
@@ -155,7 +155,7 @@ resource "kubernetes_config_map" "datasets" {
   # custom config files will be added in the future. This could take the form of a Terraform
   # resource that generates all the custom config files.
   data = {
-    "datasets.json" = "${file("${path.module}/../../hysds/configs/${var.datasets_filename}")}"
+    "datasets.json" = "${file("${path.module}/../../../hysds/configs/${var.datasets_filename}")}"
   }
 }
 
@@ -165,7 +165,7 @@ resource "kubernetes_config_map" "supervisord-job-worker" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "supervisord.conf" = "${file("${path.module}/../../hysds/factotum/supervisord.conf")}"
+    "supervisord.conf" = "${file("${path.module}/../../../hysds/factotum/supervisord.conf")}"
   }
 }
 
@@ -178,7 +178,7 @@ resource "kubernetes_config_map" "supervisord-verdi" {
   # custom config files will be added in the future. This could take the form of a Terraform
   # resource that generates all the custom config files.
   data = {
-    "supervisord.conf" = "${file("${path.module}/../../hysds/verdi/supervisord.template.conf")}"
+    "supervisord.conf" = "${file("${path.module}/../../../hysds/verdi/supervisord.template.conf")}"
   }
 }
 
@@ -188,7 +188,7 @@ resource "kubernetes_config_map" "supervisord-user-rules" {
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
   }
   data = {
-    "supervisord.conf" = "${file("${path.module}/../../hysds/user_rules/supervisord.conf")}"
+    "supervisord.conf" = "${file("${path.module}/../../../hysds/user_rules/supervisord.conf")}"
   }
 }
 
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
index 72b6c7a..1033fdf 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -476,6 +476,8 @@ resource "helm_release" "jobs-es" {
             "Stack"     = "jobs"
           }) : format("%s=%s", k, v)])
           "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
+          "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
+          "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
         }
       }
     })
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
index 0d5828e..2302861 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
@@ -50,8 +50,9 @@ resource "null_resource" "build_lambda_package" {
   }
 
   provisioner "local-exec" {
+    interpreter = ["bash", "-c"]
     command = <<EOF
-      cd ${path.module}/../../lambdas/jobs_data_ingest
+      cd ${path.module}/../../../lambdas/jobs_data_ingest
       python3.9 -m venv venv
       source venv/bin/activate
       pip install -e .
@@ -62,8 +63,8 @@ resource "null_resource" "build_lambda_package" {
       zip -r ../../lambda_package.zip .
       deactivate
       cd ../..
-      rm -rf ${path.module}/../../lambdas/jobs_data_ingest/venv
-      rm -rf ${path.module}/../../lambdas/jobs_data_ingest/lambda_package
+      rm -rf ${path.module}/../../../lambdas/jobs_data_ingest/venv
+      rm -rf ${path.module}/../../../lambdas/jobs_data_ingest/lambda_package
     EOF
   }
 }
@@ -167,7 +168,7 @@ resource "aws_lambda_function" "jobs_data_ingest" {
   timeout       = 10
 
   # Use the created ZIP file as the source of your Lambda function
-  filename = "${path.module}/../../lambdas/lambda_package.zip"
+  filename = "${path.module}/../../../lambda_package.zip"
   #   source_code_hash = filebase64sha256(pathexpand("${path.module}/../../lambdas/jobs_data_ingest/lambda_package.zip"))
 
   environment {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
index 095ecca..a22855a 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
@@ -10,6 +10,8 @@ resource "kubernetes_service" "sps-api-service" {
         "Stack"     = "spsapi"
       }) : format("%s=%s", k, v)])
       "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
+      "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
+      "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
     }
   }
   spec {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index e8f5eff..abf4816 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -195,3 +195,15 @@ variable "add_routes_to_api_gateway" {
   type        = bool
   default     = false
 }
+
+variable "lb_scheme" {
+  description = "internal or external facing loadbalancers"
+  type        = string
+  default     = "internal"
+}
+
+variable "legacy_lb_internal"{
+  description = "legacy annotation for internal lb"
+  type        = string
+  default     = "true"
+}
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/versions.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/versions.tf
index 346381a..7f3dc54 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/versions.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/versions.tf
@@ -11,7 +11,7 @@ terraform {
     }
     aws = {
       source  = "hashicorp/aws"
-      version = "4.57.1"
+      version = "~> 5.0"
     }
     null = {
       source  = "hashicorp/null"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index cff6536..5a8e2f1 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -40,6 +40,7 @@ variable "eks_cluster_name" {
 variable "kubeconfig_filepath" {
   description = "Path to the kubeconfig file for the Kubernetes cluster"
   type        = string
+  default     = "../k8s/kubernetes.yml"
 }
 
 variable "namespace" {
@@ -176,3 +177,9 @@ variable "add_routes_to_api_gateway" {
   type        = bool
   default     = false
 }
+
+variable "tags" {
+  description = "Applicable extra tags"
+  type = map(string)
+  default = {}
+}

From a59ee7b2d9c26b7d4f9fe8bf46ece9541be97556 Mon Sep 17 00:00:00 2001
From: Barber <barber@MT-306070.jpl.nasa.gov>
Date: Wed, 4 Oct 2023 09:11:18 +0100
Subject: [PATCH 31/57] add missing version update

---
 terraform-unity/versions.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-unity/versions.tf b/terraform-unity/versions.tf
index 1b563cd..3d49980 100644
--- a/terraform-unity/versions.tf
+++ b/terraform-unity/versions.tf
@@ -3,7 +3,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "4.57.1"
+      version = "~> 5.0"
     }
   }
 }

From 0270bae51e19b41c880fad8cb5444bc46d20f3a7 Mon Sep 17 00:00:00 2001
From: Barber <barber@MT-306070.jpl.nasa.gov>
Date: Wed, 4 Oct 2023 09:21:53 +0100
Subject: [PATCH 32/57] remove tags var

---
 terraform-unity/main.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 39ba645..2c691b0 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -35,5 +35,4 @@ module "unity-sps-hysds-cluster" {
   verdi_node_group_scaling_config = var.verdi_node_group_scaling_config
   verdi_node_group_instance_types = var.verdi_node_group_instance_types
   add_routes_to_api_gateway       = var.add_routes_to_api_gateway
-  tags                            = var.tags
 }

From 2ce2e524b252c7127cb7751f7e52b73a6acfc5af Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 10 Oct 2023 14:19:20 -0700
Subject: [PATCH 33/57] fix lambda path issue

---
 .../modules/terraform-unity-sps-hysds-cluster/jobs_db.tf        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
index 2302861..d11a8a2 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
@@ -168,7 +168,7 @@ resource "aws_lambda_function" "jobs_data_ingest" {
   timeout       = 10
 
   # Use the created ZIP file as the source of your Lambda function
-  filename = "${path.module}/../../../lambda_package.zip"
+  filename = "${path.module}/../../../lambdas/lambda_package.zip"
   #   source_code_hash = filebase64sha256(pathexpand("${path.module}/../../lambdas/jobs_data_ingest/lambda_package.zip"))
 
   environment {

From a3c2ae794306ab72bd8e3b3f3b00004166e3eb59 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 30 Oct 2023 13:37:04 -0700
Subject: [PATCH 34/57] sounder_sips environment test updates

---
 .../execution_post_request_body.json          | 39 ++++++++++++++
 .../L1A/test/execution_post_request_body.json |  6 +--
 .../execution_post_request_body.json          | 28 ++++++++++
 .../execution_post_request_body.json          | 52 +++++++++++++++++++
 unity-test/step_defs/conftest.py              |  6 +--
 .../step_defs/test_get_process_inputs.py      |  2 +-
 unity-test/step_defs/test_undeploy_process.py |  2 +-
 7 files changed, 127 insertions(+), 8 deletions(-)
 create mode 100644 unity-test/data/sounder_sips/L1A/sips_test/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/L1B/sips_test/execution_post_request_body.json
 create mode 100644 unity-test/data/sounder_sips/chirp/sips_test/execution_post_request_body.json

diff --git a/unity-test/data/sounder_sips/L1A/sips_test/execution_post_request_body.json b/unity-test/data/sounder_sips/L1A/sips_test/execution_post_request_body.json
new file mode 100644
index 0000000..bad1e60
--- /dev/null
+++ b/unity-test/data/sounder_sips/L1A/sips_test/execution_post_request_body.json
@@ -0,0 +1,39 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_ephatt_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:TEST1:L0_SNPP_EphAtt___1"
+    },
+    {
+      "id": "input_science_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:TEST1:L0_SNPP_ATMS_SCIENCE___1"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:TEST1:SNDR_SNPP_ATMS_L1A_OUTPUT___1"
+    },
+    {
+      "id": "static_dir",
+      "data": {
+        "class": "Directory",
+        "path": "/tmp/SOUNDER_SIPS/STATIC_DATA"
+      }
+    },
+    {
+      "id": "start_datetime",
+      "data": "2016-01-14T08:00:00Z"
+    },
+    {
+      "id": "stop_datetime",
+      "data": "2016-01-14T11:59:59Z"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json b/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json
index 9a030ba..bad1e60 100644
--- a/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json
+++ b/unity-test/data/sounder_sips/L1A/test/execution_post_request_body.json
@@ -4,15 +4,15 @@
   "inputs": [
     {
       "id": "input_ephatt_collection_id",
-      "data": "L0_SNPP_EphAtt___1"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:L0_SNPP_EphAtt___1"
     },
     {
       "id": "input_science_collection_id",
-      "data": "L0_SNPP_ATMS_SCIENCE___1"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:L0_SNPP_ATMS_SCIENCE___1"
     },
     {
       "id": "output_collection_id",
-      "data": "SNDR_SNPP_ATMS_L1A_OUTPUT___1"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:SNDR_SNPP_ATMS_L1A_OUTPUT___1"
     },
     {
       "id": "static_dir",
diff --git a/unity-test/data/sounder_sips/L1B/sips_test/execution_post_request_body.json b/unity-test/data/sounder_sips/L1B/sips_test/execution_post_request_body.json
new file mode 100644
index 0000000..598d4b7
--- /dev/null
+++ b/unity-test/data/sounder_sips/L1B/sips_test/execution_post_request_body.json
@@ -0,0 +1,28 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_collection_id",
+      "data": "SNDR_SNPP_ATMS_L1A___1"
+    },
+    {
+      "id": "start_datetime",
+      "data": "2016-01-14T08:00:00Z"
+    },
+    {
+      "id": "stop_datetime",
+      "data": "2016-01-14T11:59:59Z"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "SNDR_SNPP_ATMS_L1B_OUTPUT___1"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/data/sounder_sips/chirp/sips_test/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/sips_test/execution_post_request_body.json
new file mode 100644
index 0000000..a2be805
--- /dev/null
+++ b/unity-test/data/sounder_sips/chirp/sips_test/execution_post_request_body.json
@@ -0,0 +1,52 @@
+{
+  "mode": "async",
+  "response": "document",
+  "inputs": [
+    {
+      "id": "input_processing_labels",
+      "data": ["label1", "label2"]
+    },
+    {
+      "id": "input_cmr_collection_name",
+      "data": "C2011289787-GES_DISC"
+    },
+    {
+      "id": "input_cmr_search_start_time",
+      "data": "2016-08-22T00:10:00Z"
+    },
+    {
+      "id": "input_cmr_search_stop_time",
+      "data": "2016-08-22T00:11:00Z"
+    },
+    {
+      "id": "input_cmr_edl_user",
+      "data": "cmr_user"
+    },
+    {
+      "id": "input_cmr_edl_pass",
+      "data": "cmr_pass"
+    },
+    {
+      "id": "output_collection_id",
+      "data": "urn:nasa:unity:uds_local_test:TEST1:CHRP_16_DAY_REBIN___1"
+    },
+    {
+      "id": "output_data_bucket",
+      "data": "s3://ssips-test-ds-storage-reproc"
+    },
+    {
+      "id": "input_daac_collection_shortname",
+      "data": "CHIRP_L1B"
+    },
+    {
+      "id": "input_daac_collection_sns",
+      "data": "arn:://SNS-arn"
+    }
+  ],
+  "outputs": [
+    {
+      "id": "output",
+      "transmissionMode": "reference"
+    }
+  ]
+}
diff --git a/unity-test/step_defs/conftest.py b/unity-test/step_defs/conftest.py
index 449ac3c..ccc79ad 100644
--- a/unity-test/step_defs/conftest.py
+++ b/unity-test/step_defs/conftest.py
@@ -104,7 +104,7 @@ def _process_skip_determination(project_name, process_name, user_selected_proces
 
 
 def _undeploy_all_processes(process_service_endpoint):
-    url = urljoin(process_service_endpoint, "/processes")
+    url = urljoin(process_service_endpoint, "processes")
     get_processes_response = requests.get(url)
     get_processes_response.raise_for_status()
 
@@ -114,7 +114,7 @@ def _undeploy_all_processes(process_service_endpoint):
     for process in processes:
         url = urljoin(
             process_service_endpoint,
-            f"/processes/{process['id']}",
+            f"processes/{process['id']}",
         )
         undeploy_process_response = requests.delete(url)
         undeploy_process_response.raise_for_status()
@@ -253,7 +253,7 @@ def _request_job_execution(endpoint, process_name, request_body):
 def _request_job_status_by_id(endpoint, process_name, job_id):
     url = urljoin(
         endpoint,
-        f"/processes/{process_name.casefold()}:develop/jobs/{job_id}",
+        f"processes/{process_name.casefold()}:develop/jobs/{job_id}",
     )
     job_status_response = requests.get(url)
     job_status_response.raise_for_status()
diff --git a/unity-test/step_defs/test_get_process_inputs.py b/unity-test/step_defs/test_get_process_inputs.py
index c639ca7..a006148 100644
--- a/unity-test/step_defs/test_get_process_inputs.py
+++ b/unity-test/step_defs/test_get_process_inputs.py
@@ -23,7 +23,7 @@ def test_get_the_inputs_for_a_given_algorithm_deployment():
 def describe_process_l1b_algorithm(process_service_endpoint, project_process_dict):
     url = urljoin(
         process_service_endpoint,
-        f'/processes/{project_process_dict["process_name"].casefold()}:develop',
+        f'processes/{project_process_dict["process_name"].casefold()}:develop',
     )
     get_processes_response = requests.get(url)
     get_processes_response.raise_for_status()
diff --git a/unity-test/step_defs/test_undeploy_process.py b/unity-test/step_defs/test_undeploy_process.py
index ba77d58..4c6abc0 100644
--- a/unity-test/step_defs/test_undeploy_process.py
+++ b/unity-test/step_defs/test_undeploy_process.py
@@ -20,7 +20,7 @@ def delete_request_process_id(process_service_endpoint, project_process_dict):
     process_name = project_process_dict["process_name"]
     url = urljoin(
         process_service_endpoint,
-        f"/processes/{process_name.casefold()}:develop",
+        f"processes/{process_name.casefold()}:develop",
     )
     undeploy_process_response = requests.delete(url)
     undeploy_process_response.raise_for_status()

From 0c8a36c62f2e50565da1afcac93a2b64caa454f9 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 20 Nov 2023 10:57:29 -0800
Subject: [PATCH 35/57] add post start exec to rabbitmq container that
 increases consumer_timeout to 172800000

---
 .../terraform-unity-sps-hysds-cluster/rabbitmq.tf    | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
index 38a41df..cd77421 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
@@ -78,6 +78,16 @@ resource "kubernetes_stateful_set" "rabbitmq_statefulset" {
             name  = "RABBITMQ_ERLANG_COOKIE"
             value = "1WqgH8N2v1qDBDZDbNy8Bg9IkPWLEpu79m6q+0t36lQ="
           }
+          lifecycle {
+            post_start {
+              exec {
+                command = [
+                  "/bin/sh",
+                  "-c",
+                  <<-EOT
+                  rabbitmqctl eval 'application:set_env(rabbit, consumer_timeout, 172800000).'
+                  EOT
+                ]
           volume_mount {
             mount_path = "/var/lib/rabbitmq"
             name       = "rabbitmq-data"
@@ -93,4 +103,4 @@ resource "kubernetes_stateful_set" "rabbitmq_statefulset" {
       }
     }
   }
-}
+}
\ No newline at end of file

From 574f21167ee73dc60d3b82bbadaf1d92346e25e2 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 20 Nov 2023 11:03:21 -0800
Subject: [PATCH 36/57] close resource

---
 .../modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
index cd77421..8212f51 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
@@ -88,6 +88,9 @@ resource "kubernetes_stateful_set" "rabbitmq_statefulset" {
                   rabbitmqctl eval 'application:set_env(rabbit, consumer_timeout, 172800000).'
                   EOT
                 ]
+              }
+            }
+          }
           volume_mount {
             mount_path = "/var/lib/rabbitmq"
             name       = "rabbitmq-data"

From 99244fb2beee88889183b74b75861f44c66bc889 Mon Sep 17 00:00:00 2001
From: Luca Cinquini <cinquini@mt-509305.jpl.nasa.gov>
Date: Tue, 21 Nov 2023 11:25:25 -0700
Subject: [PATCH 37/57] Updating the parameters for tests in MCP-Test

---
 .../sounder_sips/L1B/test/execution_post_request_body.json  | 4 ++--
 .../sounder_sips/chirp/execution_post_request_body.json     | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json b/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json
index 598d4b7..ef187a7 100644
--- a/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json
+++ b/unity-test/data/sounder_sips/L1B/test/execution_post_request_body.json
@@ -4,7 +4,7 @@
   "inputs": [
     {
       "id": "input_collection_id",
-      "data": "SNDR_SNPP_ATMS_L1A___1"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:SNDR_SNPP_ATMS_L1A___1"
     },
     {
       "id": "start_datetime",
@@ -16,7 +16,7 @@
     },
     {
       "id": "output_collection_id",
-      "data": "SNDR_SNPP_ATMS_L1B_OUTPUT___1"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:SNDR_SNPP_ATMS_L1B_OUTPUT___1"
     }
   ],
   "outputs": [
diff --git a/unity-test/data/sounder_sips/chirp/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
index 54743c6..2c03c34 100644
--- a/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
+++ b/unity-test/data/sounder_sips/chirp/execution_post_request_body.json
@@ -28,15 +28,15 @@
     },
     {
       "id": "output_collection_id",
-      "data": "CHIRP_OUTPUT_COLLECTION"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:CHRP_16_DAY_REBIN___1"
     },
     {
       "id": "output_data_bucket",
-      "data": "s3://unity-data-bucket"
+      "data": "uds-test-cumulus-sps"
     },
     {
       "id": "input_daac_collection_shortname",
-      "data": "CHIRP_L1B"
+      "data": "CHIRP_OUTPUT"
     },
     {
       "id": "input_daac_collection_sns",

From 849c368a67585da8a51cf8483819d75cedfeb341 Mon Sep 17 00:00:00 2001
From: Luca Cinquini <cinquini@mt-509305.jpl.nasa.gov>
Date: Tue, 21 Nov 2023 11:27:36 -0700
Subject: [PATCH 38/57] One more update for MCP-Test tests

---
 .../chirp/test/execution_post_request_body.json             | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json b/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json
index 54743c6..2c03c34 100644
--- a/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json
+++ b/unity-test/data/sounder_sips/chirp/test/execution_post_request_body.json
@@ -28,15 +28,15 @@
     },
     {
       "id": "output_collection_id",
-      "data": "CHIRP_OUTPUT_COLLECTION"
+      "data": "urn:nasa:unity:uds_local_test:TEST1:CHRP_16_DAY_REBIN___1"
     },
     {
       "id": "output_data_bucket",
-      "data": "s3://unity-data-bucket"
+      "data": "uds-test-cumulus-sps"
     },
     {
       "id": "input_daac_collection_shortname",
-      "data": "CHIRP_L1B"
+      "data": "CHIRP_OUTPUT"
     },
     {
       "id": "input_daac_collection_sns",

From 32c1b070ee347bf42e3344f47bf60c716d7666a3 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 22 Nov 2023 12:09:11 -0800
Subject: [PATCH 39/57] working wpst private load balancer and vpc link

---
 .../ades_wpst.tf                              | 110 +++++++++++++++---
 .../outputs.tf                                |   4 +-
 2 files changed, 98 insertions(+), 16 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 2449ca8..3113d43 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -45,33 +45,115 @@ resource "kubernetes_service" "ades-wpst-api-service" {
   metadata {
     name      = "ades-wpst-api"
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
-    annotations = {
-      "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-adeswpst-RestApiLoadBalancer-${local.counter}"
-      "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
-        "Name"      = "${var.project}-${var.venue}-${var.service_area}-adeswpst-RestApiLoadBalancer-${local.counter}"
-        "Component" = "adeswpst"
-        "Stack"     = "adeswpst"
-      }) : format("%s=%s", k, v)])
-      "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-      "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
-      "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
-    }
   }
   spec {
     selector = {
       app = "ades-wpst-api"
     }
-    # type = "NodePort"
-    type = var.service_type
+    type = "NodePort"
     port {
       protocol    = "TCP"
       port        = var.service_port_map.ades_wpst_api_service
       target_port = 5000
-      # node_port   = 32000
     }
   }
 }
 
+# Define the Network Load Balancer
+resource "aws_lb" "ades-wpst-load-balancer" {
+  name               = "unity-${var.service_area}-wpst-nlb-${local.counter}"
+  internal           = true 
+  load_balancer_type = "network"
+
+  security_groups = toset([aws_security_group.shared-lb-sg.id, aws_security_group.ades-wpst-nlb-sg.id])
+  
+  # Define subnets for the NLB
+  subnets = toset(split(",", var.elb_subnets))
+}
+
+# Define a target group for the NLB
+resource "aws_lb_target_group" "ades-wpst-target-group" {
+  name     = "unity-${var.service_area}-wpst-${local.counter}-tg"
+  port     = 5000
+  protocol = "TCP"
+  target_type = "ip"
+  vpc_id   = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id  
+}
+
+resource "aws_lb_listener" "ades-wpst-k8s-service" {
+  load_balancer_arn = aws_lb.ades-wpst-load-balancer.arn
+  port = "5001"
+  protocol = "TCP"
+  default_action {
+    type = "forward"
+    target_group_arn = aws_lb_target_group.ades-wpst-target-group.arn 
+  }
+}
+
+resource "kubernetes_manifest" "ades-wpst-target-group-binding"{
+  manifest = {
+    "apiVersion" = "elbv2.k8s.aws/v1beta1"
+    "kind" = "TargetGroupBinding"
+    "metadata" = {
+      "name" = "wpst-targetgroup-binding"
+      "namespace" = kubernetes_namespace.unity-sps.metadata[0].name
+    }
+    "spec" = {
+      "serviceRef" = {
+        "name" = "ades-wpst-api"
+        "port" = var.service_port_map.ades_wpst_api_service
+      }
+      "targetGroupARN" = aws_lb_target_group.ades-wpst-target-group.arn
+    }
+  }
+}
+
+resource "aws_security_group" "shared-lb-sg"{
+  name = "${var.service_area}-shared-lb-sg-${local.counter}"
+  description = "Shared sg for all ${var.service_area} load balancers, allows creation of sg rule on cluster security group that affects all load balancers"
+  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  egress {
+    protocol = "All"
+    from_port = 0 # terraform's version of specifying "all"
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "ades-wpst-nlb-sg" {
+  name = "${var.service_area}-wpst-nlb-sg-${local.counter}"
+  description = "sg for all ${var.service_area}-wpst load balancer"
+  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  ingress {
+    protocol = "TCP"
+    from_port = var.service_port_map.ades_wpst_api_service
+    to_port = var.service_port_map.ades_wpst_api_service
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  egress {
+    protocol = "TCP"
+    from_port = 0 # terraform's version of specifying "all"
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_vpc_security_group_ingress_rule" "sps-nlb-sgr" {
+
+  security_group_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].cluster_security_group_id
+
+  description = "${var.service_area}-${local.counter} share nlb sgr, allows ingress to cluster form load balancers"
+  ip_protocol = -1 # all protocols, all ports
+  referenced_security_group_id = aws_security_group.shared-lb-sg.id # shared load balancer security group source
+}
+
+resource "aws_api_gateway_vpc_link" "ades-wpst-api-gateway-vpc-link" {
+  name = "unity-${var.service_area}-wpst-${local.counter}"
+  description = "VPC Link for ades-wpst-api load balancer"
+
+  target_arns = [aws_lb.ades-wpst-load-balancer.arn]
+}
+
 resource "kubernetes_deployment" "ades-wpst-api" {
   metadata {
     name      = "ades-wpst-api"
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
index 24490e3..d02d9b1 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
@@ -24,8 +24,8 @@ output "load_balancer_hostnames" {
   value = {
     # mozart_es     = data.kubernetes_service.mozart-es.status[0].load_balancer[0].ingress[0].hostname,
     # grq_es        = data.kubernetes_service.grq-es.status[0].load_balancer[0].ingress[0].hostname,
-    jobs_es       = data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname,
-    ades_wpst_api = kubernetes_service.ades-wpst-api-service.status[0].load_balancer[0].ingress[0].hostname,
+    # jobs_es       = data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname,
+    # ades_wpst_api = kubernetes_service.ades-wpst-api-service.status[0].load_balancer[0].ingress[0].hostname,
     sps_api       = kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname,
   }
 }

From f10614f0a75d992ac76e3e3184f2a92bcde4897f Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 27 Nov 2023 10:42:17 -0800
Subject: [PATCH 40/57] use nlbs for sps and jobs db

---
 .../ades_wpst.tf                              | 97 +------------------
 .../elastic.tf                                | 92 ++++++++----------
 .../jobs_db.tf                                |  4 +-
 .../outputs.tf                                |  6 +-
 .../sps_api.tf                                | 15 +--
 terraform-unity/outputs.tf                    |  4 +-
 6 files changed, 51 insertions(+), 167 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 3113d43..4c8613b 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -59,101 +59,6 @@ resource "kubernetes_service" "ades-wpst-api-service" {
   }
 }
 
-# Define the Network Load Balancer
-resource "aws_lb" "ades-wpst-load-balancer" {
-  name               = "unity-${var.service_area}-wpst-nlb-${local.counter}"
-  internal           = true 
-  load_balancer_type = "network"
-
-  security_groups = toset([aws_security_group.shared-lb-sg.id, aws_security_group.ades-wpst-nlb-sg.id])
-  
-  # Define subnets for the NLB
-  subnets = toset(split(",", var.elb_subnets))
-}
-
-# Define a target group for the NLB
-resource "aws_lb_target_group" "ades-wpst-target-group" {
-  name     = "unity-${var.service_area}-wpst-${local.counter}-tg"
-  port     = 5000
-  protocol = "TCP"
-  target_type = "ip"
-  vpc_id   = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id  
-}
-
-resource "aws_lb_listener" "ades-wpst-k8s-service" {
-  load_balancer_arn = aws_lb.ades-wpst-load-balancer.arn
-  port = "5001"
-  protocol = "TCP"
-  default_action {
-    type = "forward"
-    target_group_arn = aws_lb_target_group.ades-wpst-target-group.arn 
-  }
-}
-
-resource "kubernetes_manifest" "ades-wpst-target-group-binding"{
-  manifest = {
-    "apiVersion" = "elbv2.k8s.aws/v1beta1"
-    "kind" = "TargetGroupBinding"
-    "metadata" = {
-      "name" = "wpst-targetgroup-binding"
-      "namespace" = kubernetes_namespace.unity-sps.metadata[0].name
-    }
-    "spec" = {
-      "serviceRef" = {
-        "name" = "ades-wpst-api"
-        "port" = var.service_port_map.ades_wpst_api_service
-      }
-      "targetGroupARN" = aws_lb_target_group.ades-wpst-target-group.arn
-    }
-  }
-}
-
-resource "aws_security_group" "shared-lb-sg"{
-  name = "${var.service_area}-shared-lb-sg-${local.counter}"
-  description = "Shared sg for all ${var.service_area} load balancers, allows creation of sg rule on cluster security group that affects all load balancers"
-  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
-  egress {
-    protocol = "All"
-    from_port = 0 # terraform's version of specifying "all"
-    to_port = 0
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}
-
-resource "aws_security_group" "ades-wpst-nlb-sg" {
-  name = "${var.service_area}-wpst-nlb-sg-${local.counter}"
-  description = "sg for all ${var.service_area}-wpst load balancer"
-  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
-  ingress {
-    protocol = "TCP"
-    from_port = var.service_port_map.ades_wpst_api_service
-    to_port = var.service_port_map.ades_wpst_api_service
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-  egress {
-    protocol = "TCP"
-    from_port = 0 # terraform's version of specifying "all"
-    to_port = 0
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}
-
-resource "aws_vpc_security_group_ingress_rule" "sps-nlb-sgr" {
-
-  security_group_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].cluster_security_group_id
-
-  description = "${var.service_area}-${local.counter} share nlb sgr, allows ingress to cluster form load balancers"
-  ip_protocol = -1 # all protocols, all ports
-  referenced_security_group_id = aws_security_group.shared-lb-sg.id # shared load balancer security group source
-}
-
-resource "aws_api_gateway_vpc_link" "ades-wpst-api-gateway-vpc-link" {
-  name = "unity-${var.service_area}-wpst-${local.counter}"
-  description = "VPC Link for ades-wpst-api load balancer"
-
-  target_arns = [aws_lb.ades-wpst-load-balancer.arn]
-}
-
 resource "kubernetes_deployment" "ades-wpst-api" {
   metadata {
     name      = "ades-wpst-api"
@@ -290,7 +195,7 @@ resource "kubernetes_deployment" "ades-wpst-api" {
           }
           env {
             name  = "JOBS_DB_URL"
-            value = "http://${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}"
+            value = "http://${aws_lb.jobsdb-load-balancer.dns_name}:${var.service_port_map.jobs_es}"
           }
           port {
             container_port = 5000
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
index 1033fdf..88d95e0 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -359,45 +359,45 @@ locals {
   }
 }
 
-resource "null_resource" "upload_jobs_template" {
-  provisioner "local-exec" {
-    command = <<-EOT
-      set -x
-      ES_URL=${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}
-      while [[ "$(curl -s -o /dev/null -w '%%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
-      jobs_es_template='{
-        "index_patterns": ["jobs*"],
-        "template": {
-          "settings": {
-            "number_of_shards": 1,
-            "number_of_replicas": 1
-          },
-          "mappings": {
-            "dynamic": "true",
-            "properties": {
-              "id": {
-                "type": "keyword"
-              },
-              "inputs": {
-                "enabled": false
-              },
-              "outputs": {
-                "enabled": false
-              },
-              "status": {
-                "type": "keyword"
-              },
-              "labels": {
-                "enabled": false
-              }
-            }
-          }
-        }
-      }'
-      curl -X PUT "$ES_URL/_index_template/jobs_template" -H 'Content-Type: application/json' -d "$jobs_es_template"
-    EOT
-  }
-}
+# resource "null_resource" "upload_jobs_template" {
+#   provisioner "local-exec" {
+#     command = <<-EOT
+#       set -x
+#       ES_URL=${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}
+#       while [[ "$(curl -s -o /dev/null -w '%%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
+#       jobs_es_template='{
+#         "index_patterns": ["jobs*"],
+#         "template": {
+#           "settings": {
+#             "number_of_shards": 1,
+#             "number_of_replicas": 1
+#           },
+#           "mappings": {
+#             "dynamic": "true",
+#             "properties": {
+#               "id": {
+#                 "type": "keyword"
+#               },
+#               "inputs": {
+#                 "enabled": false
+#               },
+#               "outputs": {
+#                 "enabled": false
+#               },
+#               "status": {
+#                 "type": "keyword"
+#               },
+#               "labels": {
+#                 "enabled": false
+#               }
+#             }
+#           }
+#         }
+#       }'
+#       curl -X PUT "$ES_URL/_index_template/jobs_template" -H 'Content-Type: application/json' -d "$jobs_es_template"
+#     EOT
+#   }
+# }
 
 /*
 A Release is an instance of a chart running in a Kubernetes cluster.
@@ -468,18 +468,8 @@ resource "helm_release" "jobs-es" {
     yamlencode(local.jobs_es_values),
     yamlencode({
       "service" = {
-        "annotations" = {
-          "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-jobs-ElasticsearchLoadBalancer-${local.counter}"
-          "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
-            "Name"      = "${var.project}-${var.venue}-${var.service_area}-jobs-ElasticsearchLoadBalancer-${local.counter}"
-            "Component" = "jobs"
-            "Stack"     = "jobs"
-          }) : format("%s=%s", k, v)])
-          "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-          "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
-          "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
-        }
+        "type" = "NodePort"
       }
     })
   ]
-}
+}
\ No newline at end of file
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
index d11a8a2..116975f 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
@@ -175,7 +175,7 @@ resource "aws_lambda_function" "jobs_data_ingest" {
     variables = {
       REGION = var.region
       # OPENSEARCH_DOMAIN_ENDPOINT = aws_elasticsearch_domain.jobs_database.endpoint
-      ELASTICSEARCH_ENDPOINT = "http://${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}"
+      ELASTICSEARCH_ENDPOINT = "http://${aws_lb.jobsdb-load-balancer.dns_name}:${var.service_port_map.jobs_es}"
     }
   }
 
@@ -278,7 +278,7 @@ resource "aws_ssm_parameter" "jobs-db-url-param" {
   name        = "/unity/sps/${var.deployment_name}/jobsDb/url"
   description = "Full URL of the jobs db load balancer, including port for accesing jobs db"
   type        = "String"
-  value       = "http://${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}"
+  value       = "http://${aws_lb.jobsdb-load-balancer.dns_name}:${var.service_port_map.jobs_es}"
 }
 
 # resource "aws_elasticsearch_domain" "jobs_database" {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
index d02d9b1..e32aae4 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/outputs.tf
@@ -24,8 +24,8 @@ output "load_balancer_hostnames" {
   value = {
     # mozart_es     = data.kubernetes_service.mozart-es.status[0].load_balancer[0].ingress[0].hostname,
     # grq_es        = data.kubernetes_service.grq-es.status[0].load_balancer[0].ingress[0].hostname,
-    # jobs_es       = data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname,
-    # ades_wpst_api = kubernetes_service.ades-wpst-api-service.status[0].load_balancer[0].ingress[0].hostname,
-    sps_api       = kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname,
+    jobs_es       = aws_lb.jobsdb-load-balancer.dns_name,
+    ades_wpst_api = aws_lb.ades-wpst-load-balancer.dns_name,
+    sps_api       = aws_lb.sps-api-load-balancer.dns_name
   }
 }
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
index a22855a..d09213d 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
@@ -2,23 +2,12 @@ resource "kubernetes_service" "sps-api-service" {
   metadata {
     name      = "sps-api"
     namespace = kubernetes_namespace.unity-sps.metadata[0].name
-    annotations = {
-      "service.beta.kubernetes.io/aws-load-balancer-name" = "${var.project}-${var.venue}-${var.service_area}-spsapi-RestApiLoadBalancer-${local.counter}"
-      "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" = join(",", [for k, v in merge(local.common_tags, {
-        "Name"      = "${var.project}-${var.venue}-${var.service_area}-spsapi-RestApiLoadBalancer-${local.counter}"
-        "Component" = "spsapi"
-        "Stack"     = "spsapi"
-      }) : format("%s=%s", k, v)])
-      "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-      "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
-      "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
-    }
   }
   spec {
     selector = {
       app = "sps-api"
     }
-    type = var.service_type
+    type = "NodePort"
     port {
       protocol    = "TCP"
       port        = var.service_port_map.sps_api_service
@@ -31,7 +20,7 @@ resource "aws_ssm_parameter" "sps-api-hostname-param" {
   name        = "/unity/sps/${var.deployment_name}/spsApi/url"
   description = "Hostname of sps api load balancer"
   type        = "String"
-  value       = "http://${kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.sps_api_service}"
+  value       = "http://${aws_lb.sps-api-load-balancer.dns_name}:${var.service_port_map.sps_api_service}"
 }
 
 resource "kubernetes_deployment" "sps-api" {
diff --git a/terraform-unity/outputs.tf b/terraform-unity/outputs.tf
index bf164aa..e835fd9 100644
--- a/terraform-unity/outputs.tf
+++ b/terraform-unity/outputs.tf
@@ -3,8 +3,8 @@ output "load_balancer_hostnames" {
   value = {
     # mozart_es = module.unity-sps-hysds-cluster.load_balancer_hostnames.mozart_es
     # grq_es    = module.unity-sps-hysds-cluster.load_balancer_hostnames.grq_es
-    jobs_es   = module.unity-sps-hysds-cluster.load_balancer_hostnames.jobs_es
-    ades_wpst = module.unity-sps-hysds-cluster.load_balancer_hostnames.ades_wpst_api
+    # jobs_es   = module.unity-sps-hysds-cluster.load_balancer_hostnames.jobs_es
+    # ades_wpst = module.unity-sps-hysds-cluster.load_balancer_hostnames.ades_wpst_api
     sps_api   = module.unity-sps-hysds-cluster.load_balancer_hostnames.sps_api
   }
 }

From bd33a9dc5cb58db30c62e405dbf75649e965f342 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 29 Nov 2023 15:45:47 -0800
Subject: [PATCH 41/57] add aws load balancer controller to load balancer
 resources

---
 terraform-unity/.terraform.lock.hcl           |  53 +-
 .../elastic.tf                                |   3 +
 .../load_balancers.tf                         | 588 ++++++++++++++++++
 terraform-unity/outputs.tf                    |   4 +-
 4 files changed, 629 insertions(+), 19 deletions(-)
 create mode 100644 terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf

diff --git a/terraform-unity/.terraform.lock.hcl b/terraform-unity/.terraform.lock.hcl
index 92b4474..8620229 100644
--- a/terraform-unity/.terraform.lock.hcl
+++ b/terraform-unity/.terraform.lock.hcl
@@ -2,25 +2,25 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.terraform.io/hashicorp/aws" {
-  version     = "4.57.1"
-  constraints = "4.57.1"
+  version     = "5.28.0"
+  constraints = "~> 5.0"
   hashes = [
-    "h1:Qfq7Q9aCQqdl7w439mCMm89126n8DsDAmg6H8gXhnLI=",
-    "zh:44200c213ddb138df80d2a5ad86c2ebadbb5fd1d08cd7e4fc56ec6dca927659b",
-    "zh:469e6fe6a9e99e60cb168d32f05e2e9a83cf161f39160d075ff96f7674c510e1",
-    "zh:6110ba2c15a2268652ec9ea3797dd0216de84ece428055c49eaf9caa2be1ed62",
-    "zh:62ed7348acca44f64fc087e879e01cfa4e084c7600cc91e8bb7683f8065a9c79",
-    "zh:7a80e6fa9b35be178bb566093f7984dd6ffb7ad9d40b9dd5d5907f054f0c3e60",
-    "zh:8793043c8575a598c1a7cbefcb65ee1776b0061eba719098e552a3adc88f3090",
+    "h1:xIGISViKIAzm5yJ9PZejQDDxwkVMwp1CSxINPP18Fc8=",
+    "zh:062171f23f3e9d09dde4bdef4e2e1be6c10ce5392e5acb2d5674ca8d18e4efe2",
+    "zh:081f9aa09f571a95334c13eb11f7dd9e421250e5c64b2005509638eee382ccd7",
+    "zh:115f73d02f240f6626e9e4b4551dab9618a713cc238e0340155b9468b16da785",
+    "zh:1372084815a5f2e795edc1020969401786ca9032a510e0543d1e048fd699c565",
+    "zh:177a2fd380bec9fcda440d028fdf13db701d054ca637cdc860b70d62d3caafcf",
+    "zh:18274cf43f8bb0a48da25a8f511020aa4a3052582be4e48eeff4c914c0e10a31",
+    "zh:2f9d8e5b5375da4528e9ae437bbf93c2be91a50f814ca61046f3b2d16aabb3cb",
+    "zh:565a4d9e124f118fef41bd2c82e9ae3ea7316821db8f3a03838f84af7db72efb",
+    "zh:62f9f297c0ce50720e2380bd36fa1f27a210cfac08e993b0dcdb85ecf6559e07",
+    "zh:8a185766ecd16752aff72260e55e3df28a3d7e4bf28e357fbf9c0460b7ed5b39",
     "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
-    "zh:a777a0082114e273b7b3eb14095a3f6f6e703c1aff61ffb1f0846bb869e6dfc7",
-    "zh:b060c3b2973097f2087a98ac6aad7c9c89fe80f7cf3027019049feafc3f8305b",
-    "zh:e7035e74563f4486848ea1feb60852175353790bc374e0e97e241a88dc0908f7",
-    "zh:eaaa8e9eba09ada41e13116d53d4baece04fead8fcf3eab68cca3a67ed738e18",
-    "zh:ec52d8f95a84fad8fe1aae169c89d0c54d5401f75caae0869ad8182c6b6db65b",
-    "zh:f0e33174025b1b57ecfbdd09f2a59c2559ee94d7681e5ae09079e2822ec54ecf",
-    "zh:f69790a21380e5aab9303a252564737333e1e95b5d25567681630e49b17e3ec7",
-    "zh:ff6053942c40a99904bd407f3c082c1fa8f927ecce0374566eb7e8ee8145e582",
+    "zh:9b698d94915a5077d1c10a705b8d449f719eb87f25f6d46ff165b6bb9fb12778",
+    "zh:ba2c2ad8f160d9f57eaefde2171cf35697e4912f15c5cafd0ef471d1f38531f9",
+    "zh:d78d25aeed4851907817f6b281598ed853a60ca65c6bd711c8539ca3f55a841f",
+    "zh:f743437743605727edcc77c02e3a60358c222311f7a3015e883601e4e4844c1e",
   ]
 }
 
@@ -102,3 +102,22 @@ provider "registry.terraform.io/hashicorp/random" {
     "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
   ]
 }
+
+provider "registry.terraform.io/hashicorp/tls" {
+  version = "4.0.5"
+  hashes = [
+    "h1:zeG5RmggBZW/8JWIVrdaeSJa0OG62uFX5HY1eE8SjzY=",
+    "zh:01cfb11cb74654c003f6d4e32bbef8f5969ee2856394a96d127da4949c65153e",
+    "zh:0472ea1574026aa1e8ca82bb6df2c40cd0478e9336b7a8a64e652119a2fa4f32",
+    "zh:1a8ddba2b1550c5d02003ea5d6cdda2eef6870ece86c5619f33edd699c9dc14b",
+    "zh:1e3bb505c000adb12cdf60af5b08f0ed68bc3955b0d4d4a126db5ca4d429eb4a",
+    "zh:6636401b2463c25e03e68a6b786acf91a311c78444b1dc4f97c539f9f78de22a",
+    "zh:76858f9d8b460e7b2a338c477671d07286b0d287fd2d2e3214030ae8f61dd56e",
+    "zh:a13b69fb43cb8746793b3069c4d897bb18f454290b496f19d03c3387d1c9a2dc",
+    "zh:a90ca81bb9bb509063b736842250ecff0f886a91baae8de65c8430168001dad9",
+    "zh:c4de401395936e41234f1956ebadbd2ed9f414e6908f27d578614aaa529870d4",
+    "zh:c657e121af8fde19964482997f0de2d5173217274f6997e16389e7707ed8ece8",
+    "zh:d68b07a67fbd604c38ec9733069fbf23441436fecf554de6c75c032f82e1ef19",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
index 88d95e0..c0f7409 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -428,6 +428,7 @@ resource "helm_release" "mozart-es" {
       }
     })
   ]
+  depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 resource "helm_release" "grq2-es" {
@@ -454,6 +455,7 @@ resource "helm_release" "grq2-es" {
       }
     })
   ]
+  depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 resource "helm_release" "jobs-es" {
@@ -472,4 +474,5 @@ resource "helm_release" "jobs-es" {
       }
     })
   ]
+  depends_on = [helm_release.aws-load-balancer-controller]
 }
\ No newline at end of file
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
new file mode 100644
index 0000000..6f7d9aa
--- /dev/null
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
@@ -0,0 +1,588 @@
+# This security group should be added to all load balancers to allow traffic to the EKS cluster
+resource "aws_security_group" "shared-lb-sg"{
+  name = "${var.service_area}-shared-lb-sg-${local.counter}"
+  description = "Shared sg for all ${var.service_area}-${local.counter} load balancers, allows creation of sg rule on cluster security group that affects all load balancers"
+  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  egress {
+    protocol = "All"
+    from_port = 0 # terraform's version of specifying "all"
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# This security group rule adds the shared load balancer security group to the eks cluster security group so that load balancers can forward traffic to eks
+resource "aws_vpc_security_group_ingress_rule" "sps-nlb-sgr" {
+
+  security_group_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].cluster_security_group_id
+
+  description = "${var.service_area}-${local.counter} share nlb sgr, allows ingress to cluster form load balancers"
+  ip_protocol = -1 # all protocols, all ports
+  referenced_security_group_id = aws_security_group.shared-lb-sg.id # shared load balancer security group source
+}
+
+resource "aws_api_gateway_vpc_link" "ades-wpst-api-gateway-vpc-link" {
+  name = "unity-${var.service_area}-wpst-${local.counter}"
+  description = "VPC Link for ades-wpst-api load balancer"
+
+  target_arns = [aws_lb.ades-wpst-load-balancer.arn]
+}
+
+# Network Load Balancer for wpst
+resource "aws_lb" "ades-wpst-load-balancer" {
+  name               = "unity-${var.service_area}-wpst-nlb-${local.counter}"
+  internal           = true 
+  load_balancer_type = "network"
+
+  security_groups = toset([aws_security_group.shared-lb-sg.id, aws_security_group.ades-wpst-nlb-sg.id])
+  
+  # Define subnets for the NLB
+  subnets = toset(split(",", var.elb_subnets))
+}
+
+# target group for the wpst NLB
+resource "aws_lb_target_group" "ades-wpst-target-group" {
+  name     = "unity-${var.service_area}-wpst-tg-${local.counter}"
+  port     = 5000
+  protocol = "TCP"
+  target_type = "ip"
+  vpc_id   = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id  
+}
+
+# listener for the wpst NLB
+resource "aws_lb_listener" "ades-wpst-k8s-service" {
+  load_balancer_arn = aws_lb.ades-wpst-load-balancer.arn
+  port = var.service_port_map.ades_wpst_api_service
+  protocol = "TCP"
+  default_action {
+    type = "forward"
+    target_group_arn = aws_lb_target_group.ades-wpst-target-group.arn 
+  }
+}
+
+# aws load balancer controller uses TargetGroupBinding to configure the target group to use the wpst service
+resource "kubernetes_manifest" "ades-wpst-target-group-binding"{
+  manifest = {
+    "apiVersion" = "elbv2.k8s.aws/v1beta1"
+    "kind" = "TargetGroupBinding"
+    "metadata" = {
+      "name" = "wpst-targetgroup-binding"
+      "namespace" = kubernetes_namespace.unity-sps.metadata[0].name
+    }
+    "spec" = {
+      "serviceRef" = {
+        "name" = "ades-wpst-api"
+        "port" = var.service_port_map.ades_wpst_api_service
+      }
+      "targetGroupARN" = aws_lb_target_group.ades-wpst-target-group.arn
+    }
+  }
+}
+
+# wpst specific security group
+resource "aws_security_group" "ades-wpst-nlb-sg" {
+  name = "${var.service_area}-wpst-nlb-sg-${local.counter}"
+  description = "sg for all ${var.service_area}-wpst load balancer"
+  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  ingress {
+    protocol = "TCP"
+    from_port = var.service_port_map.ades_wpst_api_service
+    to_port = var.service_port_map.ades_wpst_api_service
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  egress {
+    protocol = "TCP"
+    from_port = 0 # terraform's version of specifying "all"
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_api_gateway_vpc_link" "jobsdb-gateway-vpc-link" {
+  name = "unity-${var.service_area}-jobsdb-${local.counter}"
+  description = "VPC Link for jobsdb load balancer"
+
+  target_arns = [aws_lb.jobsdb-load-balancer.arn]
+}
+
+# Network Load Balancer for jobsdb
+resource "aws_lb" "jobsdb-load-balancer" {
+  name               = "unity-${var.service_area}-jobsdb-nlb-${local.counter}"
+  internal           = true 
+  load_balancer_type = "network"
+
+  security_groups = toset([aws_security_group.shared-lb-sg.id, aws_security_group.jobsdb-nlb-sg.id])
+  
+  # Define subnets for the NLB
+  subnets = toset(split(",", var.elb_subnets))
+}
+
+# target group for the jobsdb NLB
+resource "aws_lb_target_group" "jobsdb-target-group" {
+  name     = "unity-${var.service_area}-jobsdb-tg-${local.counter}"
+  port     = 9200
+  protocol = "TCP"
+  target_type = "ip"
+  vpc_id   = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id  
+}
+
+# listener for the NLB
+resource "aws_lb_listener" "jobsdb-k8s-service" {
+  load_balancer_arn = aws_lb.jobsdb-load-balancer.arn
+  port = var.service_port_map.jobs_es
+  protocol = "TCP"
+  default_action {
+    type = "forward"
+    target_group_arn = aws_lb_target_group.jobsdb-target-group.arn 
+  }
+}
+
+# aws load balancer controller uses TargetGroupBinding to configure the target group to use the service
+resource "kubernetes_manifest" "jobsdb-target-group-binding"{
+  manifest = {
+    "apiVersion" = "elbv2.k8s.aws/v1beta1"
+    "kind" = "TargetGroupBinding"
+    "metadata" = {
+      "name" = "jobsdb-targetgroup-binding"
+      "namespace" = kubernetes_namespace.unity-sps.metadata[0].name
+    }
+    "spec" = {
+      "serviceRef" = {
+        "name" = "jobs-es"
+        "port" = var.service_port_map.jobs_es
+      }
+      "targetGroupARN" = aws_lb_target_group.jobsdb-target-group.arn
+    }
+  }
+}
+
+# lb specific security group
+resource "aws_security_group" "jobsdb-nlb-sg" {
+  name = "${var.service_area}-jobsdb-nlb-sg-${local.counter}"
+  description = "sg for ${var.service_area}-jobsdb load balancer"
+  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  ingress {
+    protocol = "TCP"
+    from_port = var.service_port_map.jobs_es
+    to_port = var.service_port_map.jobs_es
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  egress {
+    protocol = "TCP"
+    from_port = 0 # terraform's version of specifying "all"
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# Network Load Balancer for wpst
+resource "aws_lb" "sps-api-load-balancer" {
+  name               = "unity-${var.service_area}-sps-api-nlb-${local.counter}"
+  internal           = true 
+  load_balancer_type = "network"
+
+  security_groups = toset([aws_security_group.shared-lb-sg.id, aws_security_group.sps-api-nlb-sg.id])
+  
+  # Define subnets for the NLB
+  subnets = toset(split(",", var.elb_subnets))
+}
+
+# target group for the wpst NLB
+resource "aws_lb_target_group" "sps-api-target-group" {
+  name     = "unity-${var.service_area}-sps-api-tg-${local.counter}"
+  port     = 80
+  protocol = "TCP"
+  target_type = "ip"
+  vpc_id   = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id  
+}
+
+# listener for the wpst NLB
+resource "aws_lb_listener" "sps-api-k8s-service" {
+  load_balancer_arn = aws_lb.sps-api-load-balancer.arn
+  port = var.service_port_map.sps_api_service
+  protocol = "TCP"
+  default_action {
+    type = "forward"
+    target_group_arn = aws_lb_target_group.sps-api-target-group.arn 
+  }
+}
+
+# aws load balancer controller uses TargetGroupBinding to configure the target group to use the wpst service
+resource "kubernetes_manifest" "sps-api-target-group-binding"{
+  manifest = {
+    "apiVersion" = "elbv2.k8s.aws/v1beta1"
+    "kind" = "TargetGroupBinding"
+    "metadata" = {
+      "name" = "sps-api-targetgroup-binding"
+      "namespace" = kubernetes_namespace.unity-sps.metadata[0].name
+    }
+    "spec" = {
+      "serviceRef" = {
+        "name" = "sps-api"
+        "port" = var.service_port_map.sps_api_service
+      }
+      "targetGroupARN" = aws_lb_target_group.sps-api-target-group.arn
+    }
+  }
+}
+
+# wpst specific security group
+resource "aws_security_group" "sps-api-nlb-sg" {
+  name = "${var.service_area}-sps-api-nlb-sg-${local.counter}"
+  description = "sg for all ${var.service_area}-sps-api load balancer"
+  vpc_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  ingress {
+    protocol = "TCP"
+    from_port = var.service_port_map.sps_api_service
+    to_port = var.service_port_map.sps_api_service
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  egress {
+    protocol = "TCP"
+    from_port = 0 # terraform's version of specifying "all"
+    to_port = 0
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# Aws Load Balancer Controller Helm Chart
+resource "helm_release" "aws-load-balancer-controller" {
+  name = "aws-load-balancer-controller-${local.counter}"
+  repository = "https://aws.github.io/eks-charts"
+  chart = "aws-load-balancer-controller"
+  version = "1.6.1"
+  namespace = "kube-system"
+  set {
+    name = "clusterName"
+    value = data.aws_eks_cluster.sps-cluster.name
+  }
+  set {
+    name = "serviceAccount.create"
+    value = "false"
+  }
+  set {
+    name = "serviceAccount.name"
+    value = "aws-load-balancer-controller"
+  }
+  set {
+    name = "region"
+    value = var.region
+  }
+  set {
+    name = "vpcId"
+    value = data.aws_eks_cluster.sps-cluster.vpc_config[0].vpc_id
+  }
+  
+}
+
+# Create IAM OIDC provider for EKS cluster so we can add AWS Load Balancer Controller
+data "tls_certificate" "eks-cluster-oidc-server-certificate"{
+  url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
+}
+
+resource "aws_iam_openid_connect_provider" "eks-cluster-openidc-provider" {
+  url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
+
+  client_id_list = ["sts.amazon.com"]
+
+  thumbprint_list = [data.tls_certificate.eks-cluster-oidc-server-certificate.certificates[0].sha1_fingerprint]
+}
+
+resource "kubernetes_service_account" "aws-load-balancer-controller-service-account"{
+  metadata {
+    name = "aws-load-balancer-controller"
+    namespace = "kube-system"
+    annotations = {
+      "eks.amazonaws.com/role-arn": aws_iam_role.aws-load-balancer-controller-role.arn
+    }
+    labels = {
+      "app.kubernetes.io/component": "controller"
+      "app.kubernetes.io/name": "aws-load-balancer-controller"
+    }
+  }
+}
+
+locals {
+  openidc_provider_domain_name = trimprefix("https://", data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer) 
+}
+
+data "aws_iam_policy" "aws-managed-load-balancer-policy"{
+  arn = "arn:aws:iam::aws:policy/ElasticLoadBalancingFullAccess"  
+}
+
+# AwsLoadBalancerController Role and Policy from https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html
+resource "aws_iam_role" "aws-load-balancer-controller-role"{
+  name = "${var.service_area}-AwsLoadBalancerControllerRole-${local.counter}"
+
+  assume_role_policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "Federated": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.openidc_provider_domain_name}"
+            },
+            "Action": "sts:AssumeRoleWithWebIdentity",
+            "Condition": {
+                "StringEquals": {
+                    "${local.openidc_provider_domain_name}:sub": "system:serviceaccount:kube-system:aws-load-balancer-controller",
+                    "${local.openidc_provider_domain_name}:aud": "sts.amazonaws.com"
+                }
+            }
+        }
+    ]
+  })
+
+  managed_policy_arns = [aws_iam_policy.aws-load-balancer-controller-policy.arn]
+  permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/mcp-tenantOperator-AMI-APIG"
+}
+
+resource "aws_iam_role_policy_attachment" "aws-load-balancer-policy-attachment"{
+  role = aws_iam_role.aws-load-balancer-controller-role.name
+  policy_arn = data.aws_iam_policy.aws-managed-load-balancer-policy.arn
+}
+
+resource "aws_iam_policy" "aws-load-balancer-controller-policy"{
+  name = "${var.service_area}-AwsLoadBalancerControllerPolicy-${local.counter}"
+  policy = jsonencode({
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "iam:CreateServiceLinkedRole"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "StringEquals": {
+                    "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:DescribeAccountAttributes",
+                "ec2:DescribeAddresses",
+                "ec2:DescribeAvailabilityZones",
+                "ec2:DescribeInternetGateways",
+                "ec2:DescribeVpcs",
+                "ec2:DescribeVpcPeeringConnections",
+                "ec2:DescribeSubnets",
+                "ec2:DescribeSecurityGroups",
+                "ec2:DescribeInstances",
+                "ec2:DescribeNetworkInterfaces",
+                "ec2:DescribeTags",
+                "ec2:GetCoipPoolUsage",
+                "ec2:DescribeCoipPools",
+                "elasticloadbalancing:DescribeLoadBalancers",
+                "elasticloadbalancing:DescribeLoadBalancerAttributes",
+                "elasticloadbalancing:DescribeListeners",
+                "elasticloadbalancing:DescribeListenerCertificates",
+                "elasticloadbalancing:DescribeSSLPolicies",
+                "elasticloadbalancing:DescribeRules",
+                "elasticloadbalancing:DescribeTargetGroups",
+                "elasticloadbalancing:DescribeTargetGroupAttributes",
+                "elasticloadbalancing:DescribeTargetHealth",
+                "elasticloadbalancing:DescribeTags"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "cognito-idp:DescribeUserPoolClient",
+                "acm:ListCertificates",
+                "acm:DescribeCertificate",
+                "iam:ListServerCertificates",
+                "iam:GetServerCertificate",
+                "waf-regional:GetWebACL",
+                "waf-regional:GetWebACLForResource",
+                "waf-regional:AssociateWebACL",
+                "waf-regional:DisassociateWebACL",
+                "wafv2:GetWebACL",
+                "wafv2:GetWebACLForResource",
+                "wafv2:AssociateWebACL",
+                "wafv2:DisassociateWebACL",
+                "shield:GetSubscriptionState",
+                "shield:DescribeProtection",
+                "shield:CreateProtection",
+                "shield:DeleteProtection"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:RevokeSecurityGroupIngress"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateSecurityGroup"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateTags"
+            ],
+            "Resource": "arn:aws:ec2:*:*:security-group/*",
+            "Condition": {
+                "StringEquals": {
+                    "ec2:CreateAction": "CreateSecurityGroup"
+                },
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:CreateTags",
+                "ec2:DeleteTags"
+            ],
+            "Resource": "arn:aws:ec2:*:*:security-group/*",
+            "Condition": {
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:RevokeSecurityGroupIngress",
+                "ec2:DeleteSecurityGroup"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:CreateLoadBalancer",
+                "elasticloadbalancing:CreateTargetGroup"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:CreateListener",
+                "elasticloadbalancing:DeleteListener",
+                "elasticloadbalancing:CreateRule",
+                "elasticloadbalancing:DeleteRule"
+            ],
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:RemoveTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+            ],
+            "Condition": {
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags",
+                "elasticloadbalancing:RemoveTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
+                "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
+                "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
+                "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:ModifyLoadBalancerAttributes",
+                "elasticloadbalancing:SetIpAddressType",
+                "elasticloadbalancing:SetSecurityGroups",
+                "elasticloadbalancing:SetSubnets",
+                "elasticloadbalancing:DeleteLoadBalancer",
+                "elasticloadbalancing:ModifyTargetGroup",
+                "elasticloadbalancing:ModifyTargetGroupAttributes",
+                "elasticloadbalancing:DeleteTargetGroup"
+            ],
+            "Resource": "*",
+            "Condition": {
+                "Null": {
+                    "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:AddTags"
+            ],
+            "Resource": [
+                "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
+                "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
+            ],
+            "Condition": {
+                "StringEquals": {
+                    "elasticloadbalancing:CreateAction": [
+                        "CreateTargetGroup",
+                        "CreateLoadBalancer"
+                    ]
+                },
+                "Null": {
+                    "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
+                }
+            }
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:RegisterTargets",
+                "elasticloadbalancing:DeregisterTargets"
+            ],
+            "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "elasticloadbalancing:SetWebAcl",
+                "elasticloadbalancing:ModifyListener",
+                "elasticloadbalancing:AddListenerCertificates",
+                "elasticloadbalancing:RemoveListenerCertificates",
+                "elasticloadbalancing:ModifyRule"
+            ],
+            "Resource": "*"
+        }
+    ]
+  })
+}
\ No newline at end of file
diff --git a/terraform-unity/outputs.tf b/terraform-unity/outputs.tf
index e835fd9..bf164aa 100644
--- a/terraform-unity/outputs.tf
+++ b/terraform-unity/outputs.tf
@@ -3,8 +3,8 @@ output "load_balancer_hostnames" {
   value = {
     # mozart_es = module.unity-sps-hysds-cluster.load_balancer_hostnames.mozart_es
     # grq_es    = module.unity-sps-hysds-cluster.load_balancer_hostnames.grq_es
-    # jobs_es   = module.unity-sps-hysds-cluster.load_balancer_hostnames.jobs_es
-    # ades_wpst = module.unity-sps-hysds-cluster.load_balancer_hostnames.ades_wpst_api
+    jobs_es   = module.unity-sps-hysds-cluster.load_balancer_hostnames.jobs_es
+    ades_wpst = module.unity-sps-hysds-cluster.load_balancer_hostnames.ades_wpst_api
     sps_api   = module.unity-sps-hysds-cluster.load_balancer_hostnames.sps_api
   }
 }

From c74c80903f34b87776f0358efdc61e6df7c1e5a7 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Wed, 29 Nov 2023 17:29:59 -0800
Subject: [PATCH 42/57] Debugging eks 1.27 deployment

---
 terraform-unity/.terraform.lock.hcl           | 34 +++----
 .../ades_wpst.tf                              |  6 +-
 .../eks_node_groups.tf                        |  2 +-
 .../elastic.tf                                | 90 +++++++++----------
 .../sps_api.tf                                |  6 +-
 .../variables.tf                              |  6 +-
 6 files changed, 72 insertions(+), 72 deletions(-)

diff --git a/terraform-unity/.terraform.lock.hcl b/terraform-unity/.terraform.lock.hcl
index 92b4474..08e5720 100644
--- a/terraform-unity/.terraform.lock.hcl
+++ b/terraform-unity/.terraform.lock.hcl
@@ -2,25 +2,25 @@
 # Manual edits may be lost in future updates.
 
 provider "registry.terraform.io/hashicorp/aws" {
-  version     = "4.57.1"
-  constraints = "4.57.1"
+  version     = "5.23.1"
+  constraints = "~> 5.0"
   hashes = [
-    "h1:Qfq7Q9aCQqdl7w439mCMm89126n8DsDAmg6H8gXhnLI=",
-    "zh:44200c213ddb138df80d2a5ad86c2ebadbb5fd1d08cd7e4fc56ec6dca927659b",
-    "zh:469e6fe6a9e99e60cb168d32f05e2e9a83cf161f39160d075ff96f7674c510e1",
-    "zh:6110ba2c15a2268652ec9ea3797dd0216de84ece428055c49eaf9caa2be1ed62",
-    "zh:62ed7348acca44f64fc087e879e01cfa4e084c7600cc91e8bb7683f8065a9c79",
-    "zh:7a80e6fa9b35be178bb566093f7984dd6ffb7ad9d40b9dd5d5907f054f0c3e60",
-    "zh:8793043c8575a598c1a7cbefcb65ee1776b0061eba719098e552a3adc88f3090",
+    "h1:keD9rGwuFbn70D1npMx486xFsSP/TtyNa6E0AgVJY1U=",
+    "zh:024a188ad3c979a9ec0d7d898aaa90a3867a8839edc8d3543ea6155e6e010064",
+    "zh:05b73a04c58534a7527718ef55040577d5c573ea704e16a813e7d1b18a7f4c26",
+    "zh:13932cdee2fa90f40ebaa783f033752864eb6899129e055511359f8d1ada3710",
+    "zh:3500f5febc7878b4426ef89a16c0096eefd4dd0c5b0d9ba00f9ed54387df5d09",
+    "zh:394a48dea7dfb0ae40e506ccdeb5387829dbb8ab00fb64f41c347a1de092aa00",
+    "zh:51a57f258b3bce2c167b39b6ecf486f72f523da05d4c92adc6b697abe1c5ff1f",
+    "zh:7290488a96d8d10119b431eb08a37407c0812283042a21b69bcc2454eabc08ad",
+    "zh:7545389dbbba624c0ffa72fa376b359b27f484aba02139d37ee5323b589e0939",
+    "zh:92266ac6070809e0c874511ae93097c8b1eddce4c0213e487c5439e89b6ad64d",
     "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
-    "zh:a777a0082114e273b7b3eb14095a3f6f6e703c1aff61ffb1f0846bb869e6dfc7",
-    "zh:b060c3b2973097f2087a98ac6aad7c9c89fe80f7cf3027019049feafc3f8305b",
-    "zh:e7035e74563f4486848ea1feb60852175353790bc374e0e97e241a88dc0908f7",
-    "zh:eaaa8e9eba09ada41e13116d53d4baece04fead8fcf3eab68cca3a67ed738e18",
-    "zh:ec52d8f95a84fad8fe1aae169c89d0c54d5401f75caae0869ad8182c6b6db65b",
-    "zh:f0e33174025b1b57ecfbdd09f2a59c2559ee94d7681e5ae09079e2822ec54ecf",
-    "zh:f69790a21380e5aab9303a252564737333e1e95b5d25567681630e49b17e3ec7",
-    "zh:ff6053942c40a99904bd407f3c082c1fa8f927ecce0374566eb7e8ee8145e582",
+    "zh:9c3841bd650d6ba471c7159bcdfa35200e5e49c2ea11032c481a33cf7875879d",
+    "zh:bd103c46a16e7f9357e08d6427c316ccc56d203452130eed8e36ede3afa3322c",
+    "zh:cab0a16e320c6ca285a3a51f40c8f46dbaa0712856594819b415b4d8b3e63910",
+    "zh:e8adedcda4d6ff47dcae9c9bb884da26ca448fb6f7436be95ad6a341e4d8094a",
+    "zh:fc23701a3723f50878f440dcdf8768ea96d60a0d7c351aa6dfb912ad832c8384",
   ]
 }
 
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 2449ca8..8bf763a 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -52,9 +52,9 @@ resource "kubernetes_service" "ades-wpst-api-service" {
         "Component" = "adeswpst"
         "Stack"     = "adeswpst"
       }) : format("%s=%s", k, v)])
-      "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-      "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
-      "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
+      "service.beta.kubernetes.io/aws-load-balancer-subnets"  = var.elb_subnets
+      "service.beta.kubernetes.io/aws-load-balancer-scheme"   = var.lb_scheme
+      "service.beta.kubernetes.io/aws-load-balancer-external" = var.legacy_lb_external
     }
   }
   spec {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
index 538d861..6cc762a 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
@@ -15,7 +15,7 @@ locals {
 }
 
 data "aws_ssm_parameter" "mcp_linux_eks_optimized_ami" {
-  name = "/unity/account/ami/eksClusterAmi"
+  name = "/mcp/amis/aml2-eks-1-27"
 }
 
 resource "aws_iam_role" "eks_verdi_node_role" {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
index 1033fdf..40c93b3 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -359,45 +359,45 @@ locals {
   }
 }
 
-resource "null_resource" "upload_jobs_template" {
-  provisioner "local-exec" {
-    command = <<-EOT
-      set -x
-      ES_URL=${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}
-      while [[ "$(curl -s -o /dev/null -w '%%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
-      jobs_es_template='{
-        "index_patterns": ["jobs*"],
-        "template": {
-          "settings": {
-            "number_of_shards": 1,
-            "number_of_replicas": 1
-          },
-          "mappings": {
-            "dynamic": "true",
-            "properties": {
-              "id": {
-                "type": "keyword"
-              },
-              "inputs": {
-                "enabled": false
-              },
-              "outputs": {
-                "enabled": false
-              },
-              "status": {
-                "type": "keyword"
-              },
-              "labels": {
-                "enabled": false
-              }
-            }
-          }
-        }
-      }'
-      curl -X PUT "$ES_URL/_index_template/jobs_template" -H 'Content-Type: application/json' -d "$jobs_es_template"
-    EOT
-  }
-}
+# resource "null_resource" "upload_jobs_template" {
+#   provisioner "local-exec" {
+#     command = <<-EOT
+#       set -x
+#       ES_URL=${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}
+#       while [[ "$(curl -s -o /dev/null -w '%%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
+#       jobs_es_template='{
+#         "index_patterns": ["jobs*"],
+#         "template": {
+#           "settings": {
+#             "number_of_shards": 1,
+#             "number_of_replicas": 1
+#           },
+#           "mappings": {
+#             "dynamic": "true",
+#             "properties": {
+#               "id": {
+#                 "type": "keyword"
+#               },
+#               "inputs": {
+#                 "enabled": false
+#               },
+#               "outputs": {
+#                 "enabled": false
+#               },
+#               "status": {
+#                 "type": "keyword"
+#               },
+#               "labels": {
+#                 "enabled": false
+#               }
+#             }
+#           }
+#         }
+#       }'
+#       curl -X PUT "$ES_URL/_index_template/jobs_template" -H 'Content-Type: application/json' -d "$jobs_es_template"
+#     EOT
+#   }
+# }
 
 /*
 A Release is an instance of a chart running in a Kubernetes cluster.
@@ -409,7 +409,7 @@ resource "helm_release" "mozart-es" {
   namespace  = kubernetes_namespace.unity-sps.metadata[0].name
   repository = "https://helm.elastic.co"
   chart      = "elasticsearch"
-  version    = "7.9.3"
+  version    = "7.17.1"
   wait       = true
   timeout    = 600
   values = [
@@ -435,7 +435,7 @@ resource "helm_release" "grq2-es" {
   namespace  = kubernetes_namespace.unity-sps.metadata[0].name
   repository = "https://helm.elastic.co"
   chart      = "elasticsearch"
-  version    = "7.9.3"
+  version    = "7.17.1"
   wait       = true
   timeout    = 600
   values = [
@@ -461,7 +461,7 @@ resource "helm_release" "jobs-es" {
   namespace  = kubernetes_namespace.unity-sps.metadata[0].name
   repository = "https://helm.elastic.co"
   chart      = "elasticsearch"
-  version    = "7.9.3"
+  version    = "7.17.1"
   wait       = true
   timeout    = 600
   values = [
@@ -475,9 +475,9 @@ resource "helm_release" "jobs-es" {
             "Component" = "jobs"
             "Stack"     = "jobs"
           }) : format("%s=%s", k, v)])
-          "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-          "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
-          "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
+          "service.beta.kubernetes.io/aws-load-balancer-subnets"  = var.elb_subnets
+          "service.beta.kubernetes.io/aws-load-balancer-scheme"   = var.lb_scheme
+          "service.beta.kubernetes.io/aws-load-balancer-external" = var.legacy_lb_external
         }
       }
     })
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
index a22855a..4304869 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/sps_api.tf
@@ -9,9 +9,9 @@ resource "kubernetes_service" "sps-api-service" {
         "Component" = "spsapi"
         "Stack"     = "spsapi"
       }) : format("%s=%s", k, v)])
-      "service.beta.kubernetes.io/aws-load-balancer-subnets" = var.elb_subnets
-      "service.beta.kubernetes.io/aws-load-balancer-scheme" = var.lb_scheme
-      "service.beta.kubernetes.io/aws-load-balancer-internal" = var.legacy_lb_internal
+      "service.beta.kubernetes.io/aws-load-balancer-subnets"  = var.elb_subnets
+      "service.beta.kubernetes.io/aws-load-balancer-scheme"   = var.lb_scheme
+      "service.beta.kubernetes.io/aws-load-balancer-external" = var.legacy_lb_external
     }
   }
   spec {
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index abf4816..d352f5b 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -199,11 +199,11 @@ variable "add_routes_to_api_gateway" {
 variable "lb_scheme" {
   description = "internal or external facing loadbalancers"
   type        = string
-  default     = "internal"
+  default     = "external"
 }
 
-variable "legacy_lb_internal"{
-  description = "legacy annotation for internal lb"
+variable "legacy_lb_external" {
+  description = "legacy annotation for external lb"
   type        = string
   default     = "true"
 }

From d29b6a7f5db517d4a56c807e9e7379ef2da035cb Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Thu, 30 Nov 2023 15:20:04 -0800
Subject: [PATCH 43/57] fix buggy target group install

---
 .../load_balancers.tf                         | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
index 6f7d9aa..2e87369 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
@@ -75,8 +75,10 @@ resource "kubernetes_manifest" "ades-wpst-target-group-binding"{
         "port" = var.service_port_map.ades_wpst_api_service
       }
       "targetGroupARN" = aws_lb_target_group.ades-wpst-target-group.arn
+      "targetType" = "ip"
     }
   }
+  #depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 # wpst specific security group
@@ -152,8 +154,10 @@ resource "kubernetes_manifest" "jobsdb-target-group-binding"{
         "port" = var.service_port_map.jobs_es
       }
       "targetGroupARN" = aws_lb_target_group.jobsdb-target-group.arn
+      "targetType" = "ip"
     }
   }
+  #depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 # lb specific security group
@@ -175,6 +179,13 @@ resource "aws_security_group" "jobsdb-nlb-sg" {
   }
 }
 
+resource "aws_api_gateway_vpc_link" "sps-api-gateway-vpc-link" {
+  name = "unity-${var.service_area}-sps-api-${local.counter}"
+  description = "VPC Link for sps-api load balancer"
+
+  target_arns = [aws_lb.sps-api-load-balancer.arn]
+}
+
 # Network Load Balancer for wpst
 resource "aws_lb" "sps-api-load-balancer" {
   name               = "unity-${var.service_area}-sps-api-nlb-${local.counter}"
@@ -222,8 +233,10 @@ resource "kubernetes_manifest" "sps-api-target-group-binding"{
         "port" = var.service_port_map.sps_api_service
       }
       "targetGroupARN" = aws_lb_target_group.sps-api-target-group.arn
+      "targetType" = "ip"
     }
   }
+  #depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 # wpst specific security group
@@ -247,7 +260,7 @@ resource "aws_security_group" "sps-api-nlb-sg" {
 
 # Aws Load Balancer Controller Helm Chart
 resource "helm_release" "aws-load-balancer-controller" {
-  name = "aws-load-balancer-controller-${local.counter}"
+  name = "aws-load-balancer-controller"
   repository = "https://aws.github.io/eks-charts"
   chart = "aws-load-balancer-controller"
   version = "1.6.1"
@@ -283,7 +296,7 @@ data "tls_certificate" "eks-cluster-oidc-server-certificate"{
 resource "aws_iam_openid_connect_provider" "eks-cluster-openidc-provider" {
   url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
 
-  client_id_list = ["sts.amazon.com"]
+  client_id_list = ["sts.amazonaws.com"]
 
   thumbprint_list = [data.tls_certificate.eks-cluster-oidc-server-certificate.certificates[0].sha1_fingerprint]
 }
@@ -303,7 +316,7 @@ resource "kubernetes_service_account" "aws-load-balancer-controller-service-acco
 }
 
 locals {
-  openidc_provider_domain_name = trimprefix("https://", data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer) 
+  openidc_provider_domain_name = trimprefix(data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer, "https://") 
 }
 
 data "aws_iam_policy" "aws-managed-load-balancer-policy"{

From c35973daecf947a8794717d13c871d5f23a56cfc Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Thu, 30 Nov 2023 15:39:09 -0800
Subject: [PATCH 44/57] add depends on for more consistent load balancer
 deployment

---
 .../terraform-unity-sps-hysds-cluster/load_balancers.tf     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
index 2e87369..e979e94 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
@@ -78,7 +78,7 @@ resource "kubernetes_manifest" "ades-wpst-target-group-binding"{
       "targetType" = "ip"
     }
   }
-  #depends_on = [helm_release.aws-load-balancer-controller]
+  depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 # wpst specific security group
@@ -157,7 +157,7 @@ resource "kubernetes_manifest" "jobsdb-target-group-binding"{
       "targetType" = "ip"
     }
   }
-  #depends_on = [helm_release.aws-load-balancer-controller]
+  depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 # lb specific security group
@@ -236,7 +236,7 @@ resource "kubernetes_manifest" "sps-api-target-group-binding"{
       "targetType" = "ip"
     }
   }
-  #depends_on = [helm_release.aws-load-balancer-controller]
+  depends_on = [helm_release.aws-load-balancer-controller]
 }
 
 # wpst specific security group

From 3bac15a2e05a49e1924db88d62f374f011747955 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 4 Dec 2023 10:56:38 -0800
Subject: [PATCH 45/57] add account bring up module with api gw root resource

---
 .../api_gw_root_resource.tf                   | 22 +++++++++++++++++++
 .../terraform-account-bring-up/main.tf        |  6 +++++
 .../terraform-account-bring-up/variables.tf   | 11 ++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 terraform-unity/modules/terraform-account-bring-up/api_gw_root_resource.tf
 create mode 100644 terraform-unity/modules/terraform-account-bring-up/main.tf
 create mode 100644 terraform-unity/modules/terraform-account-bring-up/variables.tf

diff --git a/terraform-unity/modules/terraform-account-bring-up/api_gw_root_resource.tf b/terraform-unity/modules/terraform-account-bring-up/api_gw_root_resource.tf
new file mode 100644
index 0000000..1100418
--- /dev/null
+++ b/terraform-unity/modules/terraform-account-bring-up/api_gw_root_resource.tf
@@ -0,0 +1,22 @@
+# Rest API ID from project api gateway deployment, needed to add resources, methods, and integrations to api gateway
+data "aws_ssm_parameter" "api_gateway_rest_api_id" {
+  name  = "/unity/cs/routing/api-gateway/rest-api-id-2"
+}
+
+# Rest API root resource ID from project api gateway deployment, needed by child resources
+data "aws_api_gateway_resource" "api_gateway_rest_api_root_resource" {
+    rest_api_id = aws_ssm_parameter.api_gateway_rest_api_id.value
+    path = "/"    
+}
+
+resource "aws_api_gateway_resource" "api_gateway_sps_resource" {
+  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  parent_id   = data.aws_api_gateway_resource.api_gateway_rest_api_root_resource.id
+  path_part   = "sps"
+}
+
+resource "aws_ssm_parameter" "api_gateway_sps_path_resource_id" {
+    name = "/unity/${var.service_area}/api-gateway/sps_resource_id"
+    type = "String"
+    value = aws_api_gateway_resource.api_gateway_sps_resource.id
+}
\ No newline at end of file
diff --git a/terraform-unity/modules/terraform-account-bring-up/main.tf b/terraform-unity/modules/terraform-account-bring-up/main.tf
new file mode 100644
index 0000000..36cd662
--- /dev/null
+++ b/terraform-unity/modules/terraform-account-bring-up/main.tf
@@ -0,0 +1,6 @@
+provider "aws" {
+  region = var.region
+}
+
+data "aws_caller_identity" "current" {}
+
diff --git a/terraform-unity/modules/terraform-account-bring-up/variables.tf b/terraform-unity/modules/terraform-account-bring-up/variables.tf
new file mode 100644
index 0000000..7000f48
--- /dev/null
+++ b/terraform-unity/modules/terraform-account-bring-up/variables.tf
@@ -0,0 +1,11 @@
+variable "region" {
+  description = "The AWS region"
+  type        = string
+  default     = "us-west-2"
+}
+
+variable "service_area" {
+  description = "The service area owner of the resources being deployed"
+  type        = string
+  default     = "sps"
+}
\ No newline at end of file

From 2722c9f5cd43c2a9fce5c395656b4a1d1dac1ee7 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Mon, 4 Dec 2023 11:00:36 -0800
Subject: [PATCH 46/57] update api_gateway terraform to include sps-api, jobsdb
 and use /sps resource

---
 .../api_gateway.tf                            | 89 +++++++++++++++----
 1 file changed, 71 insertions(+), 18 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/api_gateway.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
index 38dbcbb..528c501 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/api_gateway.tf
@@ -1,13 +1,13 @@
 # Rest API ID from project api gateway deployment, needed to add resources, methods, and integrations to api gateway
 data "aws_ssm_parameter" "api_gateway_rest_api_id" {
   count = var.add_routes_to_api_gateway ? 1 : 0
-  name  = "/unity/cs/routing/api-gateway/rest-api-id"
+  name  = "/unity/cs/routing/api-gateway/rest-api-id-2"
 }
 
 # Rest API root resource ID from project api gateway deployment, needed by child resources
 data "aws_ssm_parameter" "api_gateway_rest_api_root_resource_id" {
   count = var.add_routes_to_api_gateway ? 1 : 0
-  name  = "/unity/${var.venue}/api-gateway/rest-api-root-resource-id"
+  name  = "/unity/${var.service_area}/api-gateway/sps_resource_id"
 }
 
 # Lambda authorizer ID in Rest API, needed by methods authorizing with CS custom authorizer
@@ -16,10 +16,17 @@ data "aws_ssm_parameter" "api_gateway_rest_api_lambda_authorizer_id" {
   name  = "/unity/${var.venue}/api-gateway/rest-api-lambda-authorizer-id"
 }
 
-resource "aws_api_gateway_resource" "api_gateway_ades_wpst_resource" {
+resource "aws_api_gateway_resource" "deployment_name_resource"{
   count       = var.add_routes_to_api_gateway ? 1 : 0
   rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
   parent_id   = data.aws_ssm_parameter.api_gateway_rest_api_root_resource_id[0].value
+  path_part   = var.deployment_name
+}
+
+resource "aws_api_gateway_resource" "api_gateway_ades_wpst_resource" {
+  count       = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  parent_id   = aws_api_gateway_resource.deployment_name_resource[0].id
   path_part   = "ades-wpst"
 }
 
@@ -50,31 +57,34 @@ resource "aws_api_gateway_integration" "api_gateway_ades_wpst_proxy_integration"
   http_method             = aws_api_gateway_method.api_gateway_ades_wpst_proxy_method[0].http_method
   integration_http_method = "ANY"
   type                    = "HTTP_PROXY"
-  uri                     = "http://${kubernetes_service.ades-wpst-api-service.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.ades_wpst_api_service}/{proxy}"
+  uri                     = "http://${aws_lb.ades-wpst-load-balancer.dns_name}:${var.service_port_map.ades_wpst_api_service}/{proxy}"
   cache_key_parameters    = ["method.request.path.proxy"]
   request_parameters = {
     "integration.request.path.proxy" = "method.request.path.proxy"
   }
 }
-/*
+
 resource "aws_api_gateway_resource" "api_gateway_sps_api_resource" {
-  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id.value
-  parent_id   = data.aws_ssm_parameter.api_gateway_rest_api_root_resource_id.value
+  count       = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  parent_id   = aws_api_gateway_resource.deployment_name_resource[0].id
   path_part   = "sps-api"
 }
 
 resource "aws_api_gateway_resource" "api_gateway_sps_api_proxy_resource" {
-  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id.value
-  parent_id   = aws_api_gateway_resource.api_gateway_sps_api_resource.id
+  count       = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  parent_id   = aws_api_gateway_resource.api_gateway_sps_api_resource[0].id
   path_part   = "{proxy+}"
 }
 
 resource "aws_api_gateway_method" "api_gateway_sps_api_proxy_method" {
-  rest_api_id   = data.aws_ssm_parameter.api_gateway_rest_api_id.value
-  resource_id   = aws_api_gateway_resource.api_gateway_sps_api_proxy_resource.id
+  count         = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id   = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  resource_id   = aws_api_gateway_resource.api_gateway_sps_api_proxy_resource[0].id
   http_method   = "ANY"
   authorization = "CUSTOM"
-  authorizer_id = data.aws_ssm_parameter.api_gateway_rest_api_lambda_authorizer_id.value
+  authorizer_id = data.aws_ssm_parameter.api_gateway_rest_api_lambda_authorizer_id[0].value
 
   request_parameters = {
     "method.request.path.proxy" = true
@@ -82,18 +92,59 @@ resource "aws_api_gateway_method" "api_gateway_sps_api_proxy_method" {
 }
 
 resource "aws_api_gateway_integration" "api_gateway_sps_api_proxy_integration" {
-  rest_api_id             = data.aws_ssm_parameter.api_gateway_rest_api_id.value
-  resource_id             = aws_api_gateway_resource.api_gateway_sps_api_proxy_resource.id
-  http_method             = aws_api_gateway_method.api_gateway_sps_api_proxy_method.http_method
+  count                   = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id             = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  resource_id             = aws_api_gateway_resource.api_gateway_sps_api_proxy_resource[0].id
+  http_method             = aws_api_gateway_method.api_gateway_sps_api_proxy_method[0].http_method
+  integration_http_method = "ANY"
+  type                    = "HTTP_PROXY"
+  uri                     = "http://${aws_lb.sps-api-load-balancer.dns_name}:${var.service_port_map.sps_api_service}/{proxy}"
+  cache_key_parameters    = ["method.request.path.proxy"]
+  request_parameters = {
+    "integration.request.path.proxy" = "method.request.path.proxy"
+  }
+}
+
+resource "aws_api_gateway_resource" "api_gateway_jobsdb_resource" {
+  count       = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  parent_id   = aws_api_gateway_resource.deployment_name_resource[0].id
+  path_part   = "jobs-db"
+}
+
+resource "aws_api_gateway_resource" "api_gateway_jobsdb_proxy_resource" {
+  count       = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  parent_id   = aws_api_gateway_resource.api_gateway_jobsdb_resource[0].id
+  path_part   = "{proxy+}"
+}
+
+resource "aws_api_gateway_method" "api_gateway_jobsdb_proxy_method" {
+  count         = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id   = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  resource_id   = aws_api_gateway_resource.api_gateway_jobsdb_proxy_resource[0].id
+  http_method   = "ANY"
+  authorization = "CUSTOM"
+  authorizer_id = data.aws_ssm_parameter.api_gateway_rest_api_lambda_authorizer_id[0].value
+
+  request_parameters = {
+    "method.request.path.proxy" = true
+  }
+}
+
+resource "aws_api_gateway_integration" "api_gateway_jobsdb_proxy_integration" {
+  count                   = var.add_routes_to_api_gateway ? 1 : 0
+  rest_api_id             = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
+  resource_id             = aws_api_gateway_resource.api_gateway_jobsdb_proxy_resource[0].id
+  http_method             = aws_api_gateway_method.api_gateway_jobsdb_proxy_method[0].http_method
   integration_http_method = "ANY"
   type                    = "HTTP_PROXY"
-  uri                     = "http://${kubernetes_service.sps-api-service.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.sps_api_service}/{proxy}"
+  uri                     = "http://${aws_lb.jobsdb-load-balancer.dns_name}:${var.service_port_map.jobs_es}/{proxy}"
   cache_key_parameters    = ["method.request.path.proxy"]
   request_parameters = {
     "integration.request.path.proxy" = "method.request.path.proxy"
   }
 }
-*/
 
 # Deployment resource, to enable updating a deployment when a dependent resource changes see:
 # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/api_gateway_deployment#triggers
@@ -108,7 +159,9 @@ resource "aws_api_gateway_deployment" "api_gateway_deployment" {
     venue       = var.venue
     rest_api_id = data.aws_ssm_parameter.api_gateway_rest_api_id[0].value
     redployment = sha1(jsonencode([
-      aws_api_gateway_resource.api_gateway_ades_wpst_resource[0]
+      aws_api_gateway_resource.api_gateway_ades_wpst_resource[0],
+      aws_api_gateway_resource.api_gateway_sps_api_resource[0],
+      aws_api_gateway_resource.api_gateway_jobsdb_resource[0]
     ]))
   }
   depends_on = [

From 74f693ece89c3a928f4d1b02eb7d7d1cbe29d985 Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 4 Dec 2023 13:05:52 -0800
Subject: [PATCH 47/57] Debugging eks 1.27 deployment, ebs volume var

---
 terraform-unity/main.tf                       | 59 ++++++++++---------
 .../eks_node_groups.tf                        |  2 +-
 .../variables.tf                              |  6 ++
 terraform-unity/variables.tf                  | 10 +++-
 4 files changed, 45 insertions(+), 32 deletions(-)

diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf
index 2c691b0..71b8c75 100644
--- a/terraform-unity/main.tf
+++ b/terraform-unity/main.tf
@@ -6,33 +6,34 @@
 # https://www.terraform.io/language/modules/syntax
 
 module "unity-sps-hysds-cluster" {
-  source                          = "./modules/terraform-unity-sps-hysds-cluster"
-  release                         = var.release
-  project                         = var.project == null ? data.aws_ssm_parameter.account_project[0].value : var.project
-  namespace                       = var.namespace
-  venue                           = var.venue == null ? data.aws_ssm_parameter.account_venue[0].value : var.venue
-  region                          = var.region
-  counter                         = var.counter
-  kubeconfig_filepath             = var.kubeconfig_filepath
-  docker_images                   = var.docker_images
-  service_type                    = var.service_type
-  service_port_map                = var.service_port_map
-  celeryconfig_filename           = var.celeryconfig_filename
-  datasets_filename               = var.datasets_filename
-  container_registry_server       = var.container_registry_server
-  container_registry_username     = var.container_registry_username
-  container_registry_pat          = data.aws_ssm_parameter.ghcr_pat.value
-  container_registry_owner        = var.container_registry_owner
-  uds_staging_bucket              = data.aws_ssm_parameter.uds_staging_bucket.value
-  uds_client_id                   = data.aws_ssm_parameter.uds_client_id.value
-  uds_dapa_api                    = data.aws_ssm_parameter.uds_dapa_api.value
-  uads_development_efs_fsmt_id    = var.uads_development_efs_fsmt_id == null ? data.aws_ssm_parameter.uads_development_efs_fsmt_id[0].value : var.uads_development_efs_fsmt_id
-  eks_cluster_name                = var.eks_cluster_name
-  elb_subnets                     = var.elb_subnets == null ? data.aws_ssm_parameter.eks_private_subnets[0].value : var.elb_subnets
-  default_group_node_group_name   = var.default_group_node_group_name == null ? data.aws_ssm_parameter.default_node_group_name[0].value : var.default_group_node_group_name
-  deployment_name                 = var.deployment_name
-  verdi_node_group_capacity_type  = var.verdi_node_group_capacity_type
-  verdi_node_group_scaling_config = var.verdi_node_group_scaling_config
-  verdi_node_group_instance_types = var.verdi_node_group_instance_types
-  add_routes_to_api_gateway       = var.add_routes_to_api_gateway
+  source                           = "./modules/terraform-unity-sps-hysds-cluster"
+  release                          = var.release
+  project                          = var.project == null ? data.aws_ssm_parameter.account_project[0].value : var.project
+  namespace                        = var.namespace
+  venue                            = var.venue == null ? data.aws_ssm_parameter.account_venue[0].value : var.venue
+  region                           = var.region
+  counter                          = var.counter
+  kubeconfig_filepath              = var.kubeconfig_filepath
+  docker_images                    = var.docker_images
+  service_type                     = var.service_type
+  service_port_map                 = var.service_port_map
+  celeryconfig_filename            = var.celeryconfig_filename
+  datasets_filename                = var.datasets_filename
+  container_registry_server        = var.container_registry_server
+  container_registry_username      = var.container_registry_username
+  container_registry_pat           = data.aws_ssm_parameter.ghcr_pat.value
+  container_registry_owner         = var.container_registry_owner
+  uds_staging_bucket               = data.aws_ssm_parameter.uds_staging_bucket.value
+  uds_client_id                    = data.aws_ssm_parameter.uds_client_id.value
+  uds_dapa_api                     = data.aws_ssm_parameter.uds_dapa_api.value
+  uads_development_efs_fsmt_id     = var.uads_development_efs_fsmt_id == null ? data.aws_ssm_parameter.uads_development_efs_fsmt_id[0].value : var.uads_development_efs_fsmt_id
+  eks_cluster_name                 = var.eks_cluster_name
+  elb_subnets                      = var.elb_subnets == null ? data.aws_ssm_parameter.eks_private_subnets[0].value : var.elb_subnets
+  default_group_node_group_name    = var.default_group_node_group_name == null ? data.aws_ssm_parameter.default_node_group_name[0].value : var.default_group_node_group_name
+  deployment_name                  = var.deployment_name
+  verdi_node_group_capacity_type   = var.verdi_node_group_capacity_type
+  verdi_node_group_scaling_config  = var.verdi_node_group_scaling_config
+  verdi_node_group_instance_types  = var.verdi_node_group_instance_types
+  verdi_node_group_ebs_volume_size = var.verdi_node_group_ebs_volume_size
+  add_routes_to_api_gateway        = var.add_routes_to_api_gateway
 }
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
index 6cc762a..c876e3c 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
@@ -79,7 +79,7 @@ resource "aws_launch_template" "verdi_node_group_launch_template" {
       encrypted   = false
       iops        = 3000
       throughput  = 125
-      volume_size = "80"
+      volume_size = var.verdi_node_group_ebs_volume_size
       volume_type = "gp3"
     }
   }
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index d352f5b..090f854 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -190,6 +190,12 @@ variable "verdi_node_group_instance_types" {
   default     = ["m3.medium"]
 }
 
+variable "verdi_node_group_ebs_volume_size" {
+  description = "value"
+  type        = number
+  default     = 500
+}
+
 variable "add_routes_to_api_gateway" {
   description = "If true, adds routes to api gateway configured in account"
   type        = bool
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index 5a8e2f1..27c8933 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -172,6 +172,12 @@ variable "verdi_node_group_instance_types" {
   default     = ["m3.medium"]
 }
 
+variable "verdi_node_group_ebs_volume_size" {
+  description = "value"
+  type        = number
+  default     = 500
+}
+
 variable "add_routes_to_api_gateway" {
   description = "If true, adds routes to api gateway configured in account"
   type        = bool
@@ -180,6 +186,6 @@ variable "add_routes_to_api_gateway" {
 
 variable "tags" {
   description = "Applicable extra tags"
-  type = map(string)
-  default = {}
+  type        = map(string)
+  default     = {}
 }

From 239712e204a1ae7a8ee79a403a772cc994b8a74d Mon Sep 17 00:00:00 2001
From: Drew Meyers <drew.meyers@jpl.nasa.gov>
Date: Mon, 4 Dec 2023 13:09:14 -0800
Subject: [PATCH 48/57] Add back upload jobs template

---
 .../elastic.tf                                | 78 +++++++++----------
 1 file changed, 39 insertions(+), 39 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
index 40c93b3..d467ac4 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/elastic.tf
@@ -359,45 +359,45 @@ locals {
   }
 }
 
-# resource "null_resource" "upload_jobs_template" {
-#   provisioner "local-exec" {
-#     command = <<-EOT
-#       set -x
-#       ES_URL=${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}
-#       while [[ "$(curl -s -o /dev/null -w '%%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
-#       jobs_es_template='{
-#         "index_patterns": ["jobs*"],
-#         "template": {
-#           "settings": {
-#             "number_of_shards": 1,
-#             "number_of_replicas": 1
-#           },
-#           "mappings": {
-#             "dynamic": "true",
-#             "properties": {
-#               "id": {
-#                 "type": "keyword"
-#               },
-#               "inputs": {
-#                 "enabled": false
-#               },
-#               "outputs": {
-#                 "enabled": false
-#               },
-#               "status": {
-#                 "type": "keyword"
-#               },
-#               "labels": {
-#                 "enabled": false
-#               }
-#             }
-#           }
-#         }
-#       }'
-#       curl -X PUT "$ES_URL/_index_template/jobs_template" -H 'Content-Type: application/json' -d "$jobs_es_template"
-#     EOT
-#   }
-# }
+resource "null_resource" "upload_jobs_template" {
+  provisioner "local-exec" {
+    command = <<-EOT
+      set -x
+      ES_URL=${data.kubernetes_service.jobs-es.status[0].load_balancer[0].ingress[0].hostname}:${var.service_port_map.jobs_es}
+      while [[ "$(curl -s -o /dev/null -w '%%{http_code}\n' $ES_URL)" != "200" ]]; do sleep 1; done
+      jobs_es_template='{
+        "index_patterns": ["jobs*"],
+        "template": {
+          "settings": {
+            "number_of_shards": 1,
+            "number_of_replicas": 1
+          },
+          "mappings": {
+            "dynamic": "true",
+            "properties": {
+              "id": {
+                "type": "keyword"
+              },
+              "inputs": {
+                "enabled": false
+              },
+              "outputs": {
+                "enabled": false
+              },
+              "status": {
+                "type": "keyword"
+              },
+              "labels": {
+                "enabled": false
+              }
+            }
+          }
+        }
+      }'
+      curl -X PUT "$ES_URL/_index_template/jobs_template" -H 'Content-Type: application/json' -d "$jobs_es_template"
+    EOT
+  }
+}
 
 /*
 A Release is an instance of a chart running in a Kubernetes cluster.

From 31563eb1735afc2f4411ee2db2d400ca0d77d50f Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 6 Dec 2023 10:38:40 -0800
Subject: [PATCH 49/57] comment out oidc because eks clusters already stand
 them up

---
 .../load_balancers.tf                         | 25 +++++++++++++------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
index e979e94..a406bea 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
@@ -288,18 +288,27 @@ resource "helm_release" "aws-load-balancer-controller" {
   
 }
 
+# # Custom Resource Definition for AWS Load Balancer Controller
+# data "http" "load-balancer-controller-custom-resource-definition-yaml"{
+#   url = "https://raw.githubusercontent.com/aws/eks-charts/master/stable/aws-load-balancer-controller/crds/crds.yaml"
+# }
+
+# resource "kubernetes_manifest" "load-balancer-controller-custom-resource-definition" {
+#   manifest = yamldecode(data.http.load-balancer-controller-custom-resource-definition-yaml.response_body)
+# }
+
 # Create IAM OIDC provider for EKS cluster so we can add AWS Load Balancer Controller
-data "tls_certificate" "eks-cluster-oidc-server-certificate"{
-  url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
-}
+# data "tls_certificate" "eks-cluster-oidc-server-certificate"{
+#   url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
+# }
 
-resource "aws_iam_openid_connect_provider" "eks-cluster-openidc-provider" {
-  url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
+# resource "aws_iam_openid_connect_provider" "eks-cluster-openidc-provider" {
+#   url = data.aws_eks_cluster.sps-cluster.identity[0].oidc[0].issuer
 
-  client_id_list = ["sts.amazonaws.com"]
+#   client_id_list = ["sts.amazonaws.com"]
 
-  thumbprint_list = [data.tls_certificate.eks-cluster-oidc-server-certificate.certificates[0].sha1_fingerprint]
-}
+#   thumbprint_list = [data.tls_certificate.eks-cluster-oidc-server-certificate.certificates[0].sha1_fingerprint]
+# }
 
 resource "kubernetes_service_account" "aws-load-balancer-controller-service-account"{
   metadata {

From fc3be56ebe2ad79429dd26d1ff512d3494dcac07 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 12 Dec 2023 13:50:39 -0800
Subject: [PATCH 50/57] add lb sg rule to all eks sg in cluster

---
 .../terraform-unity-sps-hysds-cluster/load_balancers.tf       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
index a406bea..7ac9bbf 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/load_balancers.tf
@@ -13,8 +13,8 @@ resource "aws_security_group" "shared-lb-sg"{
 
 # This security group rule adds the shared load balancer security group to the eks cluster security group so that load balancers can forward traffic to eks
 resource "aws_vpc_security_group_ingress_rule" "sps-nlb-sgr" {
-
-  security_group_id = data.aws_eks_cluster.sps-cluster.vpc_config[0].cluster_security_group_id
+  for_each = toset(data.aws_eks_cluster.sps-cluster.vpc_config[0].security_group_ids)
+  security_group_id = each.key
 
   description = "${var.service_area}-${local.counter} share nlb sgr, allows ingress to cluster form load balancers"
   ip_protocol = -1 # all protocols, all ports

From 52f0f347ec8b1fd2db9ac1f281f64079521a40c3 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 12 Dec 2023 13:59:31 -0800
Subject: [PATCH 51/57] add ipv4 forwarding ami fix to node group launch
 templates

---
 .../terraform-unity-sps-hysds-cluster/eks_node_groups.tf        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
index c876e3c..d8a90ca 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/eks_node_groups.tf
@@ -88,6 +88,7 @@ resource "aws_launch_template" "verdi_node_group_launch_template" {
 
   user_data = base64encode(<<-EOF
       #!/bin/bash
+      sudo sed -i 's/^net.ipv4.ip_forward = 0/net.ipv4.ip_forward = 1/' /etc/sysctl.conf && sudo sysctl -p |true
       /etc/eks/bootstrap.sh ${data.aws_eks_cluster.sps-cluster.name}
       EOF
   )
@@ -237,6 +238,7 @@ resource "aws_launch_template" "sps_api_node_group_launch_template" {
 
   user_data = base64encode(<<-EOF
       #!/bin/bash
+      sudo sed -i 's/^net.ipv4.ip_forward = 0/net.ipv4.ip_forward = 1/' /etc/sysctl.conf && sudo sysctl -p |true
       /etc/eks/bootstrap.sh ${data.aws_eks_cluster.sps-cluster.name}
       EOF
   )

From 29243680a3f30f066051c1707174286afa0b0906 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Tue, 12 Dec 2023 15:12:28 -0800
Subject: [PATCH 52/57] roll back default wpst version

---
 .../modules/terraform-unity-sps-hysds-cluster/variables.tf      | 2 +-
 terraform-unity/variables.tf                                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index 090f854..e33275f 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -63,7 +63,7 @@ variable "docker_images" {
     hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
     hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
     hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
     sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index 27c8933..9907df5 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -65,7 +65,7 @@ variable "docker_images" {
     hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
     hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
     hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
     sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"

From 3268a14c9c643e3ee231a1b80ced8a33d8f0fc90 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 13 Dec 2023 13:55:35 -0800
Subject: [PATCH 53/57] add vpc config to jobs lambda so it executes in subnets
 with jobsdb lb

---
 .../jobs_db.tf                                | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
index 116975f..21fcdbf 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/jobs_db.tf
@@ -158,6 +158,33 @@ resource "aws_iam_policy_attachment" "cloudwatch_logs_access_policy_attachment"
   policy_arn = aws_iam_policy.cloudwatch_logs_access_policy.arn
 }
 
+resource "aws_iam_policy" "lambda_ec2_policy" {
+  name        = "${var.project}-${var.venue}-${var.service_area}-IAM-EC2NetworkAccessPolicy-${local.counter}"
+  description = "Policy to allow EC2 ENI Creation for VPC constrained lambda"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = [
+          "ec2:DescribeInstances",
+          "ec2:CreateNetworkInterface",
+          "ec2:AttachNetworkInterface",
+          "ec2:DescribeNetworkInterfaces",
+          "ec2:DeleteNetworkInterface"
+        ]
+        Effect   = "Allow"
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy_attachment" "lambda_ec2_network_policy_attachment" {
+  name       = "${var.project}-${var.venue}-${var.service_area}-IAM-EC2NetworkAccessPolicyAttachment-${local.counter}"
+  roles      = [aws_iam_role.lambda_role.name]
+  policy_arn = aws_iam_policy.lambda_ec2_policy.arn
+}
 
 resource "aws_lambda_function" "jobs_data_ingest" {
   function_name = "${var.project}-${var.venue}-${var.service_area}-lambda-JobsDataIngest-${local.counter}"
@@ -170,6 +197,10 @@ resource "aws_lambda_function" "jobs_data_ingest" {
   # Use the created ZIP file as the source of your Lambda function
   filename = "${path.module}/../../../lambdas/lambda_package.zip"
   #   source_code_hash = filebase64sha256(pathexpand("${path.module}/../../lambdas/jobs_data_ingest/lambda_package.zip"))
+  vpc_config {
+    subnet_ids = split(",", var.elb_subnets)
+    security_group_ids = toset(data.aws_eks_cluster.sps-cluster.vpc_config[0].security_group_ids)
+  }
 
   environment {
     variables = {

From e1bb16761b012b3798b45eaa385884ab80881a6e Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 13 Dec 2023 14:41:56 -0800
Subject: [PATCH 54/57] use develop version of wpst

---
 .../modules/terraform-unity-sps-hysds-cluster/variables.tf      | 2 +-
 terraform-unity/variables.tf                                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index e33275f..090f854 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -63,7 +63,7 @@ variable "docker_images" {
     hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
     hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
     hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
     sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index 9907df5..27c8933 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -65,7 +65,7 @@ variable "docker_images" {
     hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
     hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
     hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.1.0"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
     sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"

From c10a6055e84c9add5ed0cad19371fceee93c1802 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Wed, 20 Dec 2023 12:50:00 -0800
Subject: [PATCH 55/57] add sleep to rabbitmq init for consistency

---
 .../rabbitmq.tf                                |  2 +-
 .../variables.tf                               | 18 +++++++++---------
 terraform-unity/variables.tf                   | 18 +++++++++---------
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
index 8212f51..f2561f7 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/rabbitmq.tf
@@ -85,7 +85,7 @@ resource "kubernetes_stateful_set" "rabbitmq_statefulset" {
                   "/bin/sh",
                   "-c",
                   <<-EOT
-                  rabbitmqctl eval 'application:set_env(rabbit, consumer_timeout, 172800000).'
+                  sleep 5; rabbitmqctl eval 'application:set_env(rabbit, consumer_timeout, 172800000).'
                   EOT
                 ]
               }
diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index 090f854..1f37728 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -57,15 +57,15 @@ variable "docker_images" {
   description = "Docker images for the Unity SPS containers"
   type        = map(string)
   default = {
-    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0"
-    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0"
-    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0"
-    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
-    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
-    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
-    sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
-    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
+    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.2.0"
+    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.2.0"
+    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.2.0"
+    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.2.0"
+    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.2.0"
+    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.2.0"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.3.0"
+    sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.2.0"
+    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.3.0"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"
     rabbitmq           = "rabbitmq:3.11.13-management"
     busybox            = "busybox:1.36.0"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index 27c8933..1b16864 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -59,15 +59,15 @@ variable "docker_images" {
   description = "Docker images for the Unity SPS containers"
   type        = map(string)
   default = {
-    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0"
-    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0"
-    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0"
-    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
-    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
-    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
-    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop"
-    sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0"
-    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"
+    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.2.0"
+    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.2.0"
+    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.2.0"
+    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.2.0"
+    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.2.0"
+    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.2.0"
+    ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.3.0"
+    sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.2.0"
+    sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.3.0"
     logstash           = "docker.elastic.co/logstash/logstash:7.10.2"
     rabbitmq           = "rabbitmq:3.11.13-management"
     busybox            = "busybox:1.36.0"

From 5c21d7b23438e91a02ca93c4465c2f9ac2ea774d Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Thu, 21 Dec 2023 10:16:14 -0800
Subject: [PATCH 56/57] fix version issue

---
 .../terraform-unity-sps-hysds-cluster/variables.tf   | 12 ++++++------
 terraform-unity/variables.tf                         | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
index 1f37728..3ab8bdb 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/variables.tf
@@ -57,12 +57,12 @@ variable "docker_images" {
   description = "Docker images for the Unity SPS containers"
   type        = map(string)
   default = {
-    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.2.0"
-    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.2.0"
-    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.2.0"
-    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.2.0"
-    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.2.0"
-    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.2.0"
+    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0"
+    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0"
+    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0"
+    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
+    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
+    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
     ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.3.0"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.2.0"
     sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.3.0"
diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf
index 1b16864..574a9e9 100644
--- a/terraform-unity/variables.tf
+++ b/terraform-unity/variables.tf
@@ -59,12 +59,12 @@ variable "docker_images" {
   description = "Docker images for the Unity SPS containers"
   type        = map(string)
   default = {
-    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.2.0"
-    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.2.0"
-    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.2.0"
-    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.2.0"
-    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.2.0"
-    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.2.0"
+    hysds_core         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0"
+    hysds_ui           = "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0"
+    hysds_mozart       = "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0"
+    hysds_grq2         = "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0"
+    hysds_verdi        = "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0"
+    hysds_factotum     = "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0"
     ades_wpst_api      = "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.3.0"
     sps_api            = "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.2.0"
     sps_hysds_pge_base = "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.3.0"

From 37361715d575954e0ca8fd74648840c5760a7d24 Mon Sep 17 00:00:00 2001
From: Ryan Hunter <ryan.g.hunter@jpl.nasa.gov>
Date: Fri, 22 Dec 2023 10:16:04 -0800
Subject: [PATCH 57/57] update wpst peristent volume recliam policy

---
 .../modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
index 4c8613b..6570a02 100644
--- a/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
+++ b/terraform-unity/modules/terraform-unity-sps-hysds-cluster/ades_wpst.tf
@@ -10,7 +10,7 @@ resource "kubernetes_persistent_volume" "ades-wpst-sqlite-pv" {
       storage = "20Gi"
     }
 
-    persistent_volume_reclaim_policy = "Delete"
+    persistent_volume_reclaim_policy = "Recycle"
 
     persistent_volume_source {
       host_path {