Merge branch 'main' into 197-deploy-cwl-airflow

.github/ISSUE_TEMPLATE/feature_request.md

@@ -29,6 +29,6 @@ Ex. I'm frustrated when [...] happens as documented in issue-XYZ
 
 **Describe the feature request**
 
-> A clear and concise description of your request. 
+> A clear and concise description of your request.
 
 Ex. I need or want [...]

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,6 +8,6 @@
 - Links to relevant issues
 - Example: issue-XYZ
 ## Testing
-- Provide some proof you've tested your changes 
+- Provide some proof you've tested your changes
 - Example: test results available at ...
 - Example: tested on operating system ...

.github/workflows/regression_test_mcp_dev.yml

@@ -13,6 +13,9 @@ on:
       SPS_API_SERVICE_ENDPOINT:
         description: "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number)"
         type: string
+      JOBS_DATABASE_ENDPOINT:
+        description: "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number)"
+        type: string
       SOUNDER_SIPS_PROCESS_SELECTION:
         description: "The Sounder SIPS process to test (L1A, L1B, chirp)"
         default: ".*"
@@ -26,6 +29,7 @@ jobs:
       - run: |
           echo "Base URL for the WPS-T API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_DEV_PROCESS_SERVICE_ENDPOINT }}"
           echo "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_DEV_SPS_API_SERVICE_ENDPOINT }}"
+          echo "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_DEV_JOBS_DATABASE_ENDPOINT }}"
           echo "The Sounder SIPS PGEs to test (L1A, L1B, chirp): ${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}"
   regression_test:
     runs-on: ubuntu-latest
@@ -46,5 +50,7 @@ jobs:
           pytest -s
           --process-service-endpoint=${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_DEV_PROCESS_SERVICE_ENDPOINT }}
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_DEV_SPS_API_SERVICE_ENDPOINT }}
+          --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_DEV_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
+          --environment "dev"
           -rs

.github/workflows/regression_test_mcp_test.yml

@@ -13,6 +13,9 @@ on:
       SPS_API_SERVICE_ENDPOINT:
         description: "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number)"
         type: string
+      JOBS_DATABASE_ENDPOINT:
+        description: "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number)"
+        type: string
       SOUNDER_SIPS_PROCESS_SELECTION:
         description: "The Sounder SIPS process to test (L1A, L1B, chirp)"
         default: ".*"
@@ -26,6 +29,7 @@ jobs:
       - run: |
           echo "Base URL for the WPS-T API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_TEST_PROCESS_SERVICE_ENDPOINT }}"
           echo "Base URL for the SPS API endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_TEST_SPS_API_SERVICE_ENDPOINT }}"
+          echo "Base URL for the Jobs DB endpoint (i.e. http://abc.def.ghi:port-number): ${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}"
           echo "The Sounder SIPS PGEs to test (L1A, L1B, chirp): ${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}"
   regression_test:
     runs-on: ubuntu-latest
@@ -46,5 +50,7 @@ jobs:
           pytest -s
           --process-service-endpoint=${{ github.event.inputs.PROCESS_SERVICE_ENDPOINT || vars.MCP_TEST_PROCESS_SERVICE_ENDPOINT }}
           --sps-api-service-endpoint=${{ github.event.inputs.SPS_API_SERVICE_ENDPOINT || vars.MCP_TEST_SPS_API_SERVICE_ENDPOINT }}
+          --jobs-database-endpoint=${{ github.event.inputs.JOBS_DATABASE_ENDPOINT || vars.MCP_TEST_JOBS_DATABASE_ENDPOINT }}
           --sounder-sips-process-selection=${{ github.event.inputs.SOUNDER_SIPS_PROCESS_SELECTION || vars.SOUNDER_SIPS_PROCESS_SELECTION }}
+          --environment "test"
           -rs

CHANGELOG.md

@@ -1,15 +1 @@
-# Changelog
-
-All notable changes to this project will be documented in this file.
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-## [0.1.0] - 2022-04-20
-
-### Added 
-
-- First implementation of HySDS services as Docker containers
-- Shell scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using the kubectl client
-- Terraform scripts to deploy/undeploy all Docker containers onto local Kubernetes cluster using Terraform
-
+Please see the parent repository's [CHANGELOG.md](https://github.com/unity-sds/unity-sps/blob/main/CHANGELOG.md) for a history of changes.

README.md

@@ -1 +1,66 @@
-# unity-sps-prototype
+<!-- Header block for project -->
+<hr>
+
+<div align="center">
+
+![logo](https://user-images.githubusercontent.com/3129134/163255685-857aa780-880f-4c09-b08c-4b53bf4af54d.png)
+<!-- ☝️ If you see logo rendering errors, make sure you're not using indentation, or try an HTML IMG tag -->
+
+<h1 align="center">Unity SPS Prototype</h1>
+<!-- ☝️ Replace with your repo name ☝️ -->
+
+</div>
+
+<pre align="center">This is a prototype repository related to Unity SPS.</pre>
+<!-- ☝️ Replace with a single sentence describing the purpose of your repo / proj ☝️ -->
+
+<!-- Header block for project -->
+
+<!-- ☝️ Add badges via: https://shields.io e.g. ![](https://img.shields.io/github/your_chosen_action/your_org/your_repo) ☝️ -->
+
+<!-- ☝️ Screenshot of your software (if applicable) via ![](https://uri-to-your-screenshot) ☝️ -->
+
+[Unity Docs](https://unity-sds.gitbook.io/docs/) | [Unity-SPS Docs](https://unity-sds.gitbook.io/docs/developer-docs/science-processing) | [Issue Tracker](https://github.com/unity-sds/unity-sps-prototype/issues)
+
+For detailed information about this repository and its purpose, please refer to the parent repository, Unity SPS, at [https://github.com/unity-sds/unity-sps](https://github.com/unity-sds/unity-sps).
+
+## Features
+
+* Please consult the parent repository [unity-sps](https://github.com/unity-sds/unity-sps) for features related to all Unity SPS repositories.
+
+<!-- ☝️ Replace with a bullet-point list of your features ☝️ -->
+
+## Contents
+
+- [Features](#features)
+- [Contents](#contents)
+- [Quick Start](#quick-start)
+- [Changelog](#changelog)
+- [Frequently Asked Questions (FAQ)](#frequently-asked-questions-faq)
+- [Contributing](#contributing)
+- [License](#license)
+- [Support](#support)
+
+## Quick Start
+
+For comprehensive details, guidelines, and documentation, refer to the parent repository: [unity-sps](https://github.com/unity-sds/unity-sps).
+
+## Changelog
+
+Please see the parent repository's [CHANGELOG.md](https://github.com/unity-sds/unity-sps/blob/main/CHANGELOG.md) for a history of changes.
+
+## Frequently Asked Questions (FAQ)
+
+For any questions, consult the FAQ of the parent repository [unity-sps](https://github.com/unity-sds/unity-sps).
+
+## Contributing
+
+To contribute, kindly refer to the guidelines in the parent repository: [unity-sps](https://github.com/unity-sds/unity-sps).
+
+## License
+
+See our: [LICENSE](LICENSE)
+
+## Support
+
+For support, please contact the maintainers of the parent repository [unity-sps](https://github.com/unity-sds/unity-sps).

hysds/configs/.netrc

@@ -1,3 +1,3 @@
 machine rabbitmq
 login guest
-password guest
+password guest

terraform-unity/README.md

@@ -188,7 +188,6 @@ $ rm terraform.tf.backup
 |------|------|
 | [aws_ssm_parameter.account_project](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.account_venue](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
-| [aws_ssm_parameter.default_group_node_group_launch_template_name](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.default_node_group_name](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.eks_private_subnets](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.ghcr_pat](https://registry.terraform.io/providers/hashicorp/aws/4.57.1/docs/data-sources/ssm_parameter) | data source |
@@ -208,10 +207,9 @@ $ rm terraform.tf.backup
 | <a name="input_container_registry_username"></a> [container\_registry\_username](#input\_container\_registry\_username) | value | `string` | `"drewm-jpl"` | no |
 | <a name="input_counter"></a> [counter](#input\_counter) | value | `string` | `""` | no |
 | <a name="input_datasets_filename"></a> [datasets\_filename](#input\_datasets\_filename) | value | `string` | `"datasets.remote.template.json"` | no |
-| <a name="input_default_group_node_group_launch_template_name"></a> [default\_group\_node\_group\_launch\_template\_name](#input\_default\_group\_node\_group\_launch\_template\_name) | value | `string` | `null` | no |
 | <a name="input_default_group_node_group_name"></a> [default\_group\_node\_group\_name](#input\_default\_group\_node\_group\_name) | value | `string` | `null` | no |
 | <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Unique name of this deployment in the account. | `string` | n/a | yes |
-| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:unity-v1.0.0",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.0.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.0.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.0.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.0.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.0.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.0.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.0.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:unity-v1.0.0"<br>}</pre> | no |
+| <a name="input_docker_images"></a> [docker\_images](#input\_docker\_images) | Docker images for the Unity SPS containers | `map(string)` | <pre>{<br>  "ades_wpst_api": "ghcr.io/unity-sds/unity-sps-prototype/ades-wpst-api:develop",<br>  "busybox": "busybox:1.36.0",<br>  "dind": "docker:23.0.3-dind",<br>  "hysds_core": "ghcr.io/unity-sds/unity-sps-prototype/hysds-core:unity-v1.1.0",<br>  "hysds_factotum": "ghcr.io/unity-sds/unity-sps-prototype/hysds-factotum:unity-v1.1.0",<br>  "hysds_grq2": "ghcr.io/unity-sds/unity-sps-prototype/hysds-grq2:unity-v1.1.0",<br>  "hysds_mozart": "ghcr.io/unity-sds/unity-sps-prototype/hysds-mozart:unity-v1.1.0",<br>  "hysds_ui": "ghcr.io/unity-sds/unity-sps-prototype/hysds-ui-remote:unity-v1.1.0",<br>  "hysds_verdi": "ghcr.io/unity-sds/unity-sps-prototype/hysds-verdi:unity-v1.1.0",<br>  "logstash": "docker.elastic.co/logstash/logstash:7.10.2",<br>  "rabbitmq": "rabbitmq:3.11.13-management",<br>  "redis": "redis:7.0.10",<br>  "sps_api": "ghcr.io/unity-sds/unity-sps-prototype/sps-api:unity-v1.1.0",<br>  "sps_hysds_pge_base": "ghcr.io/unity-sds/unity-sps-prototype/sps-hysds-pge-base:develop"<br>}</pre> | no |
 | <a name="input_eks_cluster_name"></a> [eks\_cluster\_name](#input\_eks\_cluster\_name) | The name of the EKS cluster. | `string` | n/a | yes |
 | <a name="input_elb_subnets"></a> [elb\_subnets](#input\_elb\_subnets) | value | `string` | `null` | no |
 | <a name="input_kubeconfig_filepath"></a> [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | Path to the kubeconfig file for the Kubernetes cluster | `string` | n/a | yes |
@@ -220,7 +218,7 @@ $ rm terraform.tf.backup
 | <a name="input_region"></a> [region](#input\_region) | The AWS region | `string` | `"us-west-2"` | no |
 | <a name="input_release"></a> [release](#input\_release) | The SPS release version | `string` | n/a | yes |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no |
-| <a name="input_service_port_map"></a> [service\_port\_map](#input\_service\_port\_map) | value | `map(number)` | <pre>{<br>  "ades_wpst_api_service": 5001,<br>  "grq2_es": 9201,<br>  "grq2_service": 8878,<br>  "hysds_ui_service": 3000,<br>  "mozart_es": 9200,<br>  "mozart_service": 8888,<br>  "rabbitmq_mgmt_service_cluster_rpc": 15672,<br>  "rabbitmq_service_cluster_rpc": 15672,<br>  "rabbitmq_service_epmd": 4369,<br>  "rabbitmq_service_listener": 5672,<br>  "redis_service": 6379,<br>  "sps_api_service": 5002<br>}</pre> | no |
+| <a name="input_service_port_map"></a> [service\_port\_map](#input\_service\_port\_map) | value | `map(number)` | <pre>{<br>  "ades_wpst_api_service": 5001,<br>  "grq2_es": 9201,<br>  "grq2_service": 8878,<br>  "hysds_ui_service": 3000,<br>  "jobs_es": 9202,<br>  "mozart_es": 9200,<br>  "mozart_service": 8888,<br>  "rabbitmq_mgmt_service_cluster_rpc": 15672,<br>  "rabbitmq_service_cluster_rpc": 15672,<br>  "rabbitmq_service_epmd": 4369,<br>  "rabbitmq_service_listener": 5672,<br>  "redis_service": 6379,<br>  "sps_api_service": 5002<br>}</pre> | no |
 | <a name="input_service_type"></a> [service\_type](#input\_service\_type) | value | `string` | `"LoadBalancer"` | no |
 | <a name="input_uads_development_efs_fsmt_id"></a> [uads\_development\_efs\_fsmt\_id](#input\_uads\_development\_efs\_fsmt\_id) | value | `string` | `null` | no |
 | <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | `null` | no |
@@ -233,4 +231,4 @@ $ rm terraform.tf.backup
 | Name | Description |
 |------|-------------|
 | <a name="output_load_balancer_hostnames"></a> [load\_balancer\_hostnames](#output\_load\_balancer\_hostnames) | Load Balancer Ingress Hostnames |
-<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->