Merge pull request #124 from unity-sds/updates-for-ondemand

updates needed to deploy SPS in new MCP accounts
unity-sds · Nov 11, 2022 · c472f3e · c472f3e
2 parents fa5e848 + 5da0019
commit c472f3e
Show file tree

Hide file tree

Showing 26 changed files with 277 additions and 144 deletions.
diff --git a/hysds/configs/celeryconfig_local.py b/hysds/configs/celeryconfig_local.py
@@ -1,5 +1,5 @@
-broker_url = "amqp://guest:guest@rabbitmq:5672//"
-result_backend = "redis://redis:6379"
+broker_url = "amqp://guest:guest@rabbitmq:${rabbitmq_service_port}//"
+result_backend = "redis://redis:${redis_service_port}"
 
 task_serializer = "msgpack"
 result_serializer = "msgpack"
@@ -45,7 +45,7 @@
 PYMONITOREDRUNNER_CFG = {
     "rabbitmq": {
         "hostname": "{{ MOZART_RABBIT_PVT_IP }}",
-        "port": 5672,
+        "port": ${rabbitmq_service_port},
         "queue": "stdouterr",
     },
     "StreamObserverFileWriter": {
@@ -55,24 +55,24 @@
     "StreamObserverMessenger": {"send_interval": 1},
 }
 
-MOZART_URL = "https://mozart:8888/mozart/"
-MOZART_REST_URL = "http://mozart:8888/api/v0.1"
-JOBS_ES_URL = "http://mozart-es:9200"
+MOZART_URL = "https://mozart:${mozart_service_port}/mozart/"
+MOZART_REST_URL = "http://mozart:${mozart_service_port}/api/v0.1"
+JOBS_ES_URL = "http://mozart-es:${mozart_es_port}"
 JOBS_PROCESSED_QUEUE = "jobs_processed"
 USER_RULES_JOB_QUEUE = "user_rules_job"
 ON_DEMAND_JOB_QUEUE = "on_demand_job"
 USER_RULES_JOB_INDEX = "user_rules-mozart"
 STATUS_ALIAS = "job_status"
 
 TOSCA_URL = "https://{{ GRQ_PVT_IP }}/search/"
-GRQ_URL = "http://grq2:8878"
-GRQ_REST_URL = "http://grq2:8878/api/v0.1"
-GRQ_UPDATE_URL = "http://grq2:8878/api/v0.1/grq/dataset/index"
+GRQ_URL = "http://grq2:${grq2_service_port}"
+GRQ_REST_URL = "http://grq2:${grq2_service_port}/api/v0.1"
+GRQ_UPDATE_URL = "http://grq2:${grq2_service_port}/api/v0.1/grq/dataset/index"
 
 
 GRQ_AWS_ES = False
 GRQ_ES_HOST = "grq-es"
-GRQ_ES_PORT = 9201
+GRQ_ES_PORT = ${grq2_es_port}
 GRQ_ES_PROTOCOL = "http"
 GRQ_ES_URL = "%s://%s:%d" % (GRQ_ES_PROTOCOL, GRQ_ES_HOST, GRQ_ES_PORT)
 
@@ -90,12 +90,12 @@
 
 PROCESS_EVENTS_TASKS_QUEUE = "process_events_tasks"
 
-METRICS_ES_URL = "http://{{ METRICS_ES_PVT_IP }}:9200"
+METRICS_ES_URL = "http://{{ METRICS_ES_PVT_IP }}:${mozart_es_port}"
 
 # REDIS_JOB_STATUS_URL = "redis://:{{ MOZART_REDIS_PASSWORD }}@{{ MOZART_REDIS_PVT_IP }}"
-REDIS_JOB_STATUS_URL = "redis://redis:6379"
+REDIS_JOB_STATUS_URL = "redis://redis:${redis_service_port}"
 REDIS_JOB_STATUS_KEY = "logstash"
-REDIS_JOB_INFO_URL = "redis://redis:6379"
+REDIS_JOB_INFO_URL = "redis://redis:${redis_service_port}"
 REDIS_JOB_INFO_KEY = "logstash"
 REDIS_INSTANCE_METRICS_URL = (
     "redis://:{{ METRICS_REDIS_PASSWORD }}@{{ METRICS_REDIS_PVT_IP }}"

diff --git a/hysds/configs/celeryconfig_remote.py b/hysds/configs/celeryconfig_remote.py
@@ -1,5 +1,5 @@
-broker_url = "amqp://guest:guest@rabbitmq:5672//"
-result_backend = "redis://redis:6379"
+broker_url = "amqp://guest:guest@rabbitmq:${rabbitmq_service_port}//"
+result_backend = "redis://redis:${redis_service_port}"
 # result_backend = "redis://:{{ MOZART_REDIS_PASSWORD }}@{{ MOZART_REDIS_PVT_IP }}"
 
 task_serializer = "msgpack"
@@ -46,7 +46,7 @@
 PYMONITOREDRUNNER_CFG = {
     "rabbitmq": {
         "hostname": "{{ MOZART_RABBIT_PVT_IP }}",
-        "port": 5672,
+        "port": ${rabbitmq_service_port},
         "queue": "stdouterr",
     },
     "StreamObserverFileWriter": {
@@ -56,24 +56,24 @@
     "StreamObserverMessenger": {"send_interval": 1},
 }
 
-MOZART_URL = "https://mozart:8888/mozart/"
-MOZART_REST_URL = "http://mozart:8888/api/v0.1"
-JOBS_ES_URL = "http://mozart-es:9200"
+MOZART_URL = "https://mozart:${mozart_service_port}/mozart/"
+MOZART_REST_URL = "http://mozart:${mozart_service_port}/api/v0.1"
+JOBS_ES_URL = "http://mozart-es:${mozart_es_port}"
 JOBS_PROCESSED_QUEUE = "jobs_processed"
 USER_RULES_JOB_QUEUE = "user_rules_job"
 ON_DEMAND_JOB_QUEUE = "on_demand_job"
 USER_RULES_JOB_INDEX = "user_rules-mozart"
 STATUS_ALIAS = "job_status"
 
 TOSCA_URL = "https://{{ GRQ_PVT_IP }}/search/"
-GRQ_URL = "http://grq2:8878"
-GRQ_REST_URL = "http://grq2:8878/api/v0.1"
-GRQ_UPDATE_URL = "http://grq2:8878/api/v0.1/grq/dataset/index"
+GRQ_URL = "http://grq2:${grq2_service_port}"
+GRQ_REST_URL = "http://grq2:${grq2_service_port}/api/v0.1"
+GRQ_UPDATE_URL = "http://grq2:${grq2_service_port}/api/v0.1/grq/dataset/index"
 
 
 GRQ_AWS_ES = False
 GRQ_ES_HOST = "grq-es"
-GRQ_ES_PORT = 9201
+GRQ_ES_PORT = ${grq2_es_port}
 GRQ_ES_PROTOCOL = "http"
 GRQ_ES_URL = "%s://%s:%d" % (GRQ_ES_PROTOCOL, GRQ_ES_HOST, GRQ_ES_PORT)
 
@@ -91,12 +91,12 @@
 
 PROCESS_EVENTS_TASKS_QUEUE = "process_events_tasks"
 
-METRICS_ES_URL = "http://{{ METRICS_ES_PVT_IP }}:9200"
+METRICS_ES_URL = "http://{{ METRICS_ES_PVT_IP }}:${mozart_es_port}"
 
 # REDIS_JOB_STATUS_URL = "redis://:{{ MOZART_REDIS_PASSWORD }}@{{ MOZART_REDIS_PVT_IP }}"
-REDIS_JOB_STATUS_URL = "redis://redis:6379"
+REDIS_JOB_STATUS_URL = "redis://redis:${redis_service_port}"
 REDIS_JOB_STATUS_KEY = "logstash"
-REDIS_JOB_INFO_URL = "redis://redis:6379"
+REDIS_JOB_INFO_URL = "redis://redis:${redis_service_port}"
 REDIS_JOB_INFO_KEY = "logstash"
 REDIS_INSTANCE_METRICS_URL = (
     "redis://:{{ METRICS_REDIS_PASSWORD }}@{{ METRICS_REDIS_PVT_IP }}"

diff --git a/hysds/configs/datasets.json b/hysds/configs/datasets.json
@@ -9,18 +9,18 @@
       "type": "area_of_interest",
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/products/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/browse/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/{id}"
         ]
       }
     },
@@ -33,18 +33,18 @@
       "extractor": null,
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       }
     }

diff --git a/hysds/configs/datasets.local.template.json b/hysds/configs/datasets.local.template.json
@@ -9,18 +9,18 @@
       "type": "area_of_interest",
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/products/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/browse/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/{id}"
         ]
       }
     },
@@ -33,18 +33,18 @@
       "extractor": null,
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       }
     }

diff --git a/hysds/configs/datasets.remote.template.json b/hysds/configs/datasets.remote.template.json
@@ -9,18 +9,18 @@
       "type": "area_of_interest",
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/{id}",
         "urls": [
-          "http://minio:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/products/{id}"
+          "http://minio:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/{id}",
         "urls": [
-          "http://minio:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/browse/{id}"
+          "http://minio:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/{id}"
         ]
       }
     },
@@ -33,18 +33,18 @@
       "extractor": null,
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://minio:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://minio:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://minio:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://minio:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       }
     }

diff --git a/hysds/configs/datasets.template.json b/hysds/configs/datasets.template.json
@@ -9,18 +9,18 @@
       "type": "area_of_interest",
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/products/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/{id}",
-          "s3://minio:9000/datasets/browse/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/{id}"
         ]
       }
     },
@@ -33,18 +33,18 @@
       "extractor": null,
       "publish": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/products/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       },
       "browse": {
         "s3-profile-name": "default",
-        "location": "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
+        "location": "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}",
         "urls": [
-          "http://localhost:9001/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
-          "s3://minio:9000/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
+          "http://localhost:${minio_service_interface_port}/buckets/datasets/browse/products/hello_world/{version}/{year}/{month}/{day}/{id}",
+          "s3://minio:${minio_service_api_port}/datasets/browse/hello_world/{version}/{year}/{month}/{day}/{id}"
         ]
       }
     }

diff --git a/hysds/grq/rest_api/settings.cfg b/hysds/grq/rest_api/settings.cfg
@@ -2,12 +2,11 @@
 AWS_ES = False
 ES_PROTOCOL = "http"
 ES_HOST = "grq-es"
-ES_PORT = 9201
-# ES_URL = "{{ GRQ_ES_PROTOCOL or 'http' }}://{{ GRQ_ES_PVT_IP }}:{{ GRQ_ES_PORT or 9200}}"
-ES_URL = "http://grq-es:9201"
+ES_PORT = ${grq2_es_port}
+ES_URL = "http://grq-es:${grq2_es_port}"
 
 # Mozart Elasticsearch URL
-MOZART_ES_URL = "http://mozart-es:9200"
+MOZART_ES_URL = "http://mozart-es:${mozart_es_port}"
 
 # alias to ElasticSearch GRQ index
 GRQ_INDEX = "grq"
@@ -16,7 +15,7 @@ GRQ_INDEX = "grq"
 GEONAMES_INDEX = "geonames"
 
 # Redis URL
-REDIS_URL = "redis://redis:6379/0"
+REDIS_URL = "redis://redis:${redis_service_port}/0"
 
 # ES index for user rules (located in mozart's ES)
 USER_RULES_INDEX = "user_rules-grq"