Merge branch 'local.es' into api-documentation

cumulus_lambda_functions/cumulus_es_setup/es_setup.py

@@ -16,9 +16,10 @@ def __init__(self):
         required_env = ['ES_URL']
         if not all([k in os.environ for k in required_env]):
             raise EnvironmentError(f'one or more missing env: {required_env}')
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+        self.__es: ESAbstract = ESFactory().get_instance(os.getenv('ES_TYPE', 'AWS'),
                                                          index=DBConstants.collections_index,
                                                          base_url=os.getenv('ES_URL'),
+                                                         use_ssl=os.getenv('ES_USE_SSL', 'TRUE').strip() is True,
                                                          port=int(os.getenv('ES_PORT', '443'))
                                                          )
 

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -132,6 +132,40 @@ def query_direct_to_private_api(self, private_api_prefix: str, transform=True):
             return {'server_error': f'error while invoking:{str(e)}'}
         return {'results': stac_list}
 
+    def add_entry(self, private_api_prefix: str, new_granule: dict):
+        raise NotImplementedError(f'Please implement adding granules to Cumulus')
+        # https://nasa.github.io/cumulus-api/v18.4.0/#create-granule
+        # payload = {
+        #     'httpMethod': 'POST',
+        #     'resource': '/{proxy+}',
+        #     'path': f'/{self.__collections_key}',
+        #     'headers': {
+        #         'Content-Type': 'application/json',
+        #     },
+        #     'body': json.dumps(new_granule)
+        # }
+        # LOGGER.debug(f'payload: {payload}')
+        # try:
+        #     query_result = self._invoke_api(payload, private_api_prefix)
+        #     """
+        #     {'statusCode': 500, 'body': '', 'headers': {}}
+        #     """
+        #     if query_result['statusCode'] >= 500:
+        #         LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}')
+        #         return {'server_error': query_result}
+        #     if query_result['statusCode'] >= 400:
+        #         LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}')
+        #         return {'client_error': query_result}
+        #     query_result = json.loads(query_result['body'])
+        #     LOGGER.debug(f'json query_result: {query_result}')
+        #     if 'message' not in query_result:
+        #         return {'server_error': f'invalid response: {query_result}'}
+        # except Exception as e:
+        #     LOGGER.exception('error while invoking')
+        #     return {'server_error': f'error while invoking:{str(e)}'}
+        # return {'status': query_result['message']}
+        return
+
     def delete_entry(self, private_api_prefix: str, granule_id: str):
         payload = {
             'httpMethod': 'DELETE',

cumulus_lambda_functions/daac_archiver/daac_archiver_logic.py

@@ -23,7 +23,7 @@
 class DaacArchiverLogic:
     def __init__(self):
         self.__es_url, self.__es_port = os.getenv('ES_URL'), int(os.getenv('ES_PORT', '443'))
-        self.__archive_index_logic = UdsArchiveConfigIndex(self.__es_url, self.__es_port)
+        self.__archive_index_logic = UdsArchiveConfigIndex(self.__es_url, self.__es_port, os.getenv('ES_TYPE', 'AWS'), os.getenv('ES_USE_SSL', 'TRUE').strip() is True)
         self.__granules_index = GranulesDbIndex()
         self.__sns = AwsSns()
         self.__s3 = AwsS3()

cumulus_lambda_functions/granules_cnm_ingester/granules_cnm_ingester_logic.py

@@ -40,7 +40,7 @@ def __init__(self):
         if 'UNITY_DEFAULT_PROVIDER' not in os.environ:
             raise ValueError(f'missing UNITY_DEFAULT_PROVIDER')
         self.__default_provider = os.environ.get('UNITY_DEFAULT_PROVIDER')
-        self.__uds_collection = UdsCollections(es_url=os.getenv('ES_URL'), es_port=int(os.getenv('ES_PORT', '443')))
+        self.__uds_collection = UdsCollections(es_url=os.getenv('ES_URL'), es_port=int(os.getenv('ES_PORT', '443')), es_type=os.getenv('ES_TYPE', 'AWS'))
 
     @property
     def successful_features_json(self):

cumulus_lambda_functions/lib/authorization/uds_authorizer_es_identity_pool.py

@@ -12,11 +12,12 @@
 
 class UDSAuthorizorEsIdentityPool(UDSAuthorizorAbstract):
 
-    def __init__(self, es_url: str, es_port=443) -> None:
+    def __init__(self, es_url: str, es_port=443, es_type='AWS', use_ssl=True) -> None:
         super().__init__()
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+        self.__es: ESAbstract = ESFactory().get_instance(es_type,
                                                          index=DBConstants.authorization_index,
                                                          base_url=es_url,
+                                                         use_ssl=use_ssl,
                                                          port=es_port)
 
     def add_authorized_group(self, action: [str], resource: [str], tenant: str, venue: str, ldap_group_name: str):

cumulus_lambda_functions/lib/authorization/uds_authorizer_factory.py

@@ -1,12 +1,18 @@
+import os
+
 from mdps_ds_lib.lib.utils.factory_abstract import FactoryAbstract
 
 
 class UDSAuthorizerFactory(FactoryAbstract):
     cognito = 'COGNITO'
 
     def get_instance(self, class_type, **kwargs):
+        if 'use_ssl' not in kwargs:
+            kwargs['use_ssl'] = os.getenv('ES_USE_SSL', 'TRUE').strip() is True
+        if 'es_type' not in kwargs:
+            kwargs['es_type'] = os.getenv('ES_TYPE', 'AWS')
         if class_type == self.cognito:
             from cumulus_lambda_functions.lib.authorization.uds_authorizer_es_identity_pool import \
                 UDSAuthorizorEsIdentityPool
-            return UDSAuthorizorEsIdentityPool(kwargs['es_url'], kwargs['es_port'])
+            return UDSAuthorizorEsIdentityPool(kwargs['es_url'], kwargs['es_port'], kwargs['es_type'], kwargs['use_ssl'])
         raise ValueError(f'class_type: {class_type} not implemented')

cumulus_lambda_functions/lib/uds_db/archive_index.py

@@ -25,10 +25,12 @@ class UdsArchiveConfigIndex:
             'archiving_types': {'type': 'array', 'items': {'type': 'object'}},
         }
     }
-    def __init__(self, es_url, es_port=443):
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+
+    def __init__(self, es_url, es_port=443, es_type='AWS', use_ssl=True):
+        self.__es: ESAbstract = ESFactory().get_instance(es_type,
                                                          index='TODO',
                                                          base_url=es_url,
+                                                         use_ssl=use_ssl,
                                                          port=es_port)
         self.__tenant, self.__venue = '', ''
 

cumulus_lambda_functions/lib/uds_db/granules_db_index.py

@@ -19,10 +19,11 @@ def __init__(self):
         required_env = ['ES_URL']
         if not all([k in os.environ for k in required_env]):
             raise EnvironmentError(f'one or more missing env: {required_env}')
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+        self.__es: ESAbstract = ESFactory().get_instance(os.getenv('ES_TYPE', 'AWS'),
                                                          index=DBConstants.collections_index,
                                                          base_url=os.getenv('ES_URL'),
-                                                         port=int(os.getenv('ES_PORT', '443'))
+                                                         port=int(os.getenv('ES_PORT', '443')),
+                                                         use_ssl=os.getenv('ES_USE_SSL', 'TRUE').strip() is True,
                                                          )
         # self.__default_fields = {
         #     "granule_id": {"type": "keyword"},
@@ -36,8 +37,11 @@ def __init__(self):
     def to_es_bbox(bbox_array):
         # lon = x, lat = y
         # lon, lat, lon, lat
+        # -180, -90, 180, 90
         # x can be 170 to -170
         # 170, 0, -170, 10
+        # latitude must be between -90.0 and 90.0
+        # longitude must be between -180.0 and 180.0
         minX, minY, maxX, maxY = bbox_array
 
         # Ensure the values are properly sorted

cumulus_lambda_functions/lib/uds_db/uds_collections.py

@@ -21,10 +21,11 @@ class UdsCollections:
     start_time = 'start_time'
     end_time = 'end_time'
 
-    def __init__(self, es_url, es_port=443):
-        self.__es: ESAbstract = ESFactory().get_instance('AWS',
+    def __init__(self, es_url, es_port=443, es_type='AWS', use_ssl=True):
+        self.__es: ESAbstract = ESFactory().get_instance(es_type,
                                                          index=DBConstants.collections_index,
                                                          base_url=es_url,
+                                                         use_ssl=use_ssl,
                                                          port=es_port)
 
     @staticmethod

cumulus_lambda_functions/uds_api/catalog_api.py

@@ -39,7 +39,7 @@ async def get_catalog(request: Request, limit: Union[int, None] = 10, offset: Un
                       )
     auth_info = FastApiUtils.get_authorization_info(request)
     uds_collections = UdsCollections(es_url=os.getenv('ES_URL'),
-                                     es_port=int(os.getenv('ES_PORT', '443')))
+                                     es_port=int(os.getenv('ES_PORT', '443')), es_type=os.getenv('ES_TYPE', 'AWS'))
     collection_regexes = authorizer.get_authorized_collections(DBConstants.read, auth_info['ldap_groups'])
     LOGGER.info(f'collection_regexes: {collection_regexes}')
     authorized_collections = uds_collections.get_collections(collection_regexes)

cumulus_lambda_functions/uds_api/collections_api.py

@@ -142,6 +142,7 @@ async def create_new_collection(request: Request, new_collection: CumulusCollect
 
 
 @router.post("/actual")
+@router.post("/actual/")
 async def create_new_collection_real(request: Request, new_collection: CumulusCollectionModel):
     """
     Actual endpoint to create a new Cumulus Collection
@@ -192,7 +193,7 @@ async def get_single_collection(request: Request, collection_id: str, limit: Uni
                       )
     auth_info = FastApiUtils.get_authorization_info(request)
     uds_collections = UdsCollections(es_url=os.getenv('ES_URL'),
-                                     es_port=int(os.getenv('ES_PORT', '443')))
+                                     es_port=int(os.getenv('ES_PORT', '443')), es_type=os.getenv('ES_TYPE', 'AWS'))
     if collection_id is None or collection_id == '':
         raise HTTPException(status_code=500, detail=f'missing or invalid collection_id: {collection_id}')
     collection_identifier = uds_collections.decode_identifier(collection_id)
@@ -237,7 +238,7 @@ async def query_collections(request: Request, collection_id: Union[str, None] =
                       )
     auth_info = FastApiUtils.get_authorization_info(request)
     uds_collections = UdsCollections(es_url=os.getenv('ES_URL'),
-                                     es_port=int(os.getenv('ES_PORT', '443')))
+                                     es_port=int(os.getenv('ES_PORT', '443')), es_type=os.getenv('ES_TYPE', 'AWS'))
     if collection_id is not None:
         collection_identifier = uds_collections.decode_identifier(collection_id)
         if not authorizer.is_authorized_for_collection(DBConstants.read, collection_id,

cumulus_lambda_functions/uds_api/dapa/auth_crud.py

@@ -97,7 +97,7 @@ def __init__(self, authorization_info, request_body):
         self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \
             .get_instance(UDSAuthorizerFactory.cognito,
                           es_url=self.__es_url,
-                          es_port=self.__es_port
+                          es_port=self.__es_port, es_type=os.getenv('ES_TYPE', 'AWS')
                           )
 
     def is_admin(self):