feat: add new lambda for collections

Author: wphyojpl

cumulus_lambda_functions/cumulus_collections_dapa/__init__.py

@@ -0,0 +1,60 @@
+import os
+
+from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
+
+
+class CumulusCollectionsDapa:
+    def __init__(self, event):
+        LOGGER.info(f'event: {event}')
+        self.__event = event
+        self.__jwt_token = 'NA'
+        self.__limit = 10
+        self.__offset = 0
+        self.__page_number = (self.__offset // self.__limit) + 1
+        if 'CUMULUS_BASE' not in os.environ:
+            raise EnvironmentError('missing key: CUMULUS_BASE')
+        if 'CUMULUS_LAMBDA_PREFIX' not in os.environ:
+            raise EnvironmentError('missing key: CUMULUS_LAMBDA_PREFIX')
+
+        self.__cumulus_base = os.getenv('CUMULUS_BASE')
+        self.__cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX')
+
+        self.__cumulus = CollectionsQuery(self.__cumulus_base, self.__jwt_token)
+        self.__cumulus.with_limit(self.__limit)
+        self.__cumulus.with_page_number(self.__page_number)
+
+    def __assign_values(self):
+        if 'queryStringParameters' not in self.__event:
+            return self
+        query_str_dict = self.__event['queryStringParameters']
+        if 'limit' in query_str_dict:
+            self.__limit = int(query_str_dict['limit'])
+        if 'offset' in query_str_dict:
+            self.__offset = int(query_str_dict['offset'])
+        return self
+
+    def start(self):
+        try:
+            cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
+        except Exception as e:
+            return {
+                'statusCode': 500,
+                'body': {'message': f'unpredicted error: {str(e)}'}
+            }
+        if 'server_error' in cumulus_result:
+            return {
+                'statusCode': 500,
+                'body': {'message': cumulus_result['server_error']}
+            }
+        if 'client_error' in cumulus_result:
+            return {
+                'statusCode': 400,
+                'body': {'message': cumulus_result['client_error']}
+            }
+        return {
+            'statusCode': 200,
+            'body': json.dumps({'features': cumulus_result['results']})
+        }

cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py

@@ -0,0 +1,14 @@
+from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_collections_dapa import CumulusCollectionsDapa
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+
+def lambda_handler(event, context):
+    """
+{'cma': {'task_config': {'bucket': '{$.meta.buckets.internal.name}', 'collection': '{$.meta.collection}', 'cumulus_message': {'outputs': [{'source': '{$.files}', 'destination': '{$.payload}'}]}}, 'event': {'cumulus_meta': {'cumulus_version': '10.0.1', 'execution_name': 'c6d885dc-b4b2-4eb0-b22e-b6f58a7a0870', 'message_source': 'sfn', 'queueExecutionLimits': {'https://sqs.us-west-2.amazonaws.com/884500545225/am-uds-dev-cumulus-backgroundProcessing': 5}, 'state_machine': 'arn:aws:states:us-west-2:884500545225:stateMachine:am-uds-dev-cumulus-IngestGranule', 'system_bucket': 'am-uds-dev-cumulus-internal', 'workflow_start_time': 1646785175509, 'parentExecutionArn': 'arn:aws:states:us-west-2:884500545225:execution:am-uds-dev-cumulus-DiscoverGranules:885483b4-ba55-4db1-b197-661e1e595a45', 'queueUrl': 'arn:aws:sqs:us-west-2:884500545225:am-uds-dev-cumulus-startSF'}, 'exception': 'None', 'meta': {'buckets': {'internal': {'name': 'am-uds-dev-cumulus-internal', 'type': 'internal'}, 'protected': {'name': 'am-uds-dev-cumulus-protected', 'type': 'protected'}}, 'cmr': {'clientId': 'CHANGEME', 'cmrEnvironment': 'UAT', 'cmrLimit': 100, 'cmrPageSize': 50, 'oauthProvider': 'earthdata', 'passwordSecretName': 'am-uds-dev-cumulus-message-template-cmr-password20220216072916956000000002', 'provider': 'CHANGEME', 'username': 'username'}, 'collection': {'name': 'ATMS_SCIENCE_Group_2011', 'version': '001', 'process': 'modis', 'granuleId': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$', 'granuleIdExtraction': '(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS', 'duplicateHandling': 'replace', 'url_path': '{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}', 'provider_path': '/data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/', 'files': [{'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000000.PDS', 'type': 'data'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS', 'type': 'metadata'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS.xml', 'type': 'metadata'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0\\.cmr\\.xml$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS.xml', 'type': 'metadata'}], 'updatedAt': 1646326197526, 'createdAt': 1646258167624}, 'distribution_endpoint': 's3://am-uds-dev-cumulus-internal/', 'launchpad': {'api': 'launchpadApi', 'certificate': 'launchpad.pfx', 'passphraseSecretName': ''}, 'provider': {'password': 'AQICAHhSagsGDAl5tQWM010IEvxKgj2LcsNub5v5FHoRpOjXcQHFbE4iMnF/W0Y/NrsYvrfHAAAAajBoBgkqhkiG9w0BBwagWzBZAgEAMFQGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMLaH13SdxPXREjXLtAgEQgCfA+lEu2c/xLTGwJsbtKlXJbKDy4pwV+rS3BnJqgBoLLMQZqOdoFhk=', 'host': 'snppl0.gesdisc.eosdis.nasa.gov', 'updatedAt': 1646244053419, 'protocol': 'https', 'createdAt': 1646244053419, 'encrypted': True, 'username': 'AQICAHhSagsGDAl5tQWM010IEvxKgj2LcsNub5v5FHoRpOjXcQGRoY5EBMpvvyMASUowBM61AAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM9OhRHwTuxiz74q4UAgEQgCDEHOhsVG6+LqXfnlw+Z3Wg9MDOCd9/K5/X5j3tPJYkaA==', 'allowedRedirects': ['https://urs.earthdata.nasa.gov', 'urs.earthdata.nasa.gov'], 'id': 'snpp_provider_02', 'globalConnectionLimit': 10}, 'stack': 'am-uds-dev-cumulus', 'template': 's3://am-uds-dev-cumulus-internal/am-uds-dev-cumulus/workflow_template.json', 'workflow_name': 'IngestGranule', 'workflow_tasks': {'SyncGranule': {'name': 'am-uds-dev-cumulus-SyncGranule', 'version': '$LATEST', 'arn': 'arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-SyncGranule'}}, 'staticValue': 'aStaticValue', 'interpolatedValueStackName': 'am-uds-dev-cumulus', 'input_granules': [{'granuleId': 'P1570515ATMSSCIENCEAXT1134912000000', 'dataType': 'ATMS_SCIENCE_Group_2011', 'version': '001', 'files': [{'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000000.PDS', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000000.PDS', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000000.PDS', 'type': 'data', 'size': 744}, {'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000001.PDS', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000001.PDS', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000001.PDS', 'type': 'metadata', 'size': 18084600}, {'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'type': 'metadata', 'size': 9526}], 'sync_granule_duration': 9822, 'createdAt': 1647386972717}], 'process': 'modis'}, 'payload': {}, 'replace': {'Bucket': 'am-uds-dev-cumulus-internal', 'Key': 'events/5d8edf37-0a18-4af5-a76f-7c2091cdd1e2', 'TargetPath': '$.payload'}}}}
+    :param event:
+    :param context:
+    :return:
+    """
+    LambdaLoggerGenerator.remove_default_handlers()
+    # TODO implement
+    return CumulusCollectionsDapa(event).start()

cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py

@@ -1,3 +1,6 @@
+import boto3
+import json
+
 from abc import ABC
 from copy import deepcopy
 
@@ -22,6 +25,17 @@ def with_limit(self, limit: int):
     def get_base_headers(self):
         return deepcopy(self.__base_headers)
 
+    def _invoke_api(self, payload, private_api_prefix: str):
+        """Function to invoke cumulus api via aws lambda"""
+        client = boto3.client('lambda')
+        response = client.invoke(
+            FunctionName=f'{private_api_prefix}-PrivateApiLambda',
+            Payload=json.dumps(payload),
+        )
+        json_response_payload = response.get('Payload').read().decode('utf-8')
+        response_data = json.loads(json_response_payload)
+        return response_data
+
     @property
     def cumulus_base(self):
         return self.__cumulus_base

cumulus_lambda_functions/cumulus_wrapper/cumulus_base.py

@@ -0,0 +1,68 @@
+import json
+
+import requests
+from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer
+
+from cumulus_lambda_functions.cumulus_wrapper.cumulus_base import CumulusBase
+from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
+
+LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
+
+
+class CollectionsQuery(CumulusBase):
+    __collections_key = 'collections'
+
+    def __init__(self, cumulus_base: str, cumulus_token: str):
+        super().__init__(cumulus_base, cumulus_token)
+
+    def query_direct_to_private_api(self, private_api_prefix: str):
+        payload = {
+            'httpMethod': 'GET',
+            'resource': '/{proxy+}',
+            'path': f'/{self.__collections_key}',
+            'queryStringParameters': {k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]},
+        }
+        LOGGER.debug(f'payload: {payload}')
+        try:
+            query_result = self._invoke_api(payload, private_api_prefix)
+            """
+        {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False}
+            """
+            if query_result['statusCode'] >= 500:
+                LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}')
+                return {'server_error': query_result}
+            if query_result['statusCode'] >= 400:
+                LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}')
+                return {'client_error': query_result}
+            query_result = json.loads(query_result['body'])
+            LOGGER.info(f'json query_result: {query_result}')
+            if 'results' not in query_result:
+                LOGGER.error(f'missing key: results. invalid response json: {query_result}')
+                return {'server_error': f'missing key: results. invalid response json: {query_result}'}
+            query_result = query_result['results']
+            stac_list = [CollectionTransformer().to_stac(k) for k in query_result]
+        except Exception as e:
+            LOGGER.exception('error while invoking')
+            return {'server_error': f'error while invoking:{str(e)}'}
+        return {'results': stac_list}
+
+    def query(self):
+        conditions_str = '&'.join(self._conditions)
+        LOGGER.info(f'cumulus_base: {self.cumulus_base}')
+        LOGGER.info(f'get_base_headers: {self.get_base_headers()}')
+        try:
+            query_result = requests.get(url=f'{self.cumulus_base}/{self.__collections_key}?{conditions_str}', headers=self.get_base_headers())
+            LOGGER.info(f'query_result: {query_result}')
+            if query_result.status_code >= 500:
+                return {'server_error': query_result.text}
+            if query_result.status_code >= 400:
+                return {'client_error': query_result.text}
+            query_result = json.loads(query_result.content.decode())
+            LOGGER.info(f'query_result: {query_result}')
+            if 'results' not in query_result:
+                return {'server_error': f'missing key: results. invalid response json: {query_result}'}
+            query_result = query_result['results']
+        except Exception as e:
+            LOGGER.exception('error during cumulus query')
+            return {'server_error': str(e)}
+        return {'results': [CollectionTransformer().to_stac(k) for k in query_result]}

cumulus_lambda_functions/cumulus_wrapper/query_collections.py

@@ -51,19 +51,6 @@ def with_time_range(self, from_time, to_time):
         self._conditions.append(f'{self.__beginning_time_key}__to={to_time}')
         return self
 
-    def __invoke_api(self, payload, private_api_prefix: str):
-        """Function to invoke cumulus api via aws lambda"""
-
-        client = boto3.client('lambda')
-        response = client.invoke(
-            FunctionName=f'{private_api_prefix}-PrivateApiLambda',
-            Payload=json.dumps(payload),
-        )
-        json_response_payload = response.get('Payload').read().decode('utf-8')
-        response_data = json.loads(json_response_payload)
-
-        return response_data
-
     def query_direct_to_private_api(self, private_api_prefix: str):
         payload = {
             'httpMethod': 'GET',
@@ -78,7 +65,7 @@ def query_direct_to_private_api(self, private_api_prefix: str):
         }
         LOGGER.debug(f'payload: {payload}')
         try:
-            query_result = self.__invoke_api(payload, private_api_prefix)
+            query_result = self._invoke_api(payload, private_api_prefix)
             """
         {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False}
             """

cumulus_lambda_functions/cumulus_wrapper/query_granules.py

@@ -32,10 +32,6 @@ paths:
         in: "query"
         required: false
         type: "string"
-      - name: "Authorization"
-        in: "header"
-        required: false
-        type: "string"
       responses:
         "200":
           description: "200 response"
@@ -55,7 +51,75 @@ paths:
 securityDefinitions:
   Unity_API_Gateway_Lambda_WPST_UI_Demo_Authorizer_Node:
     type: "apiKey"
-    name: "Authorization"
+    name: "Authorization"---
+            swagger: "2.0"
+            info:
+              description: "Unity API Gateway"
+              version: "2022-05-04T20:23:33Z"
+              title: "Unity API Gateway"
+            host: "k3a3qmarxh.execute-api.us-west-2.amazonaws.com"
+            basePath: "/dev"
+            schemes:
+              - "https"
+            paths:
+              /am-uds-dapa/collections/{collectionId}/observation/items:
+                get:
+                  produces:
+                    - "application/json"
+                  parameters:
+                    - name: "collectionId"
+                      in: "path"
+                    - name: "datetime"
+                      in: "query"
+                      required: false
+                      type: "string"
+                    - name: "limit"
+                      in: "query"
+                      required: false
+                      type: "string"
+                    - name: "offset"
+                      in: "query"
+                      required: false
+                      type: "string"
+                    - name: "bbox"
+                      in: "query"
+                      required: false
+                      type: "string"
+                  responses:
+                    "200":
+                      description: "200 response"
+                      schema:
+                        $ref: "#/definitions/Empty"
+                  x-amazon-apigateway-request-validator: "Validate body, query string parameters,\
+                                 \ and headers"
+                  x-amazon-apigateway-integration:
+                    type: "aws_proxy"
+                    httpMethod: "POST"
+                    uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:am-uds-dapa-test-1/invocations"
+                    responses:
+                      default:
+                        statusCode: "200"
+                    passthroughBehavior: "when_no_match"
+                    contentHandling: "CONVERT_TO_TEXT"
+            securityDefinitions:
+              Unity_API_Gateway_Lambda_WPST_UI_Demo_Authorizer_Node:
+                type: "apiKey"
+                name: "Authorization"
+                in: "header"
+                x-amazon-apigateway-authtype: "custom"
+                x-amazon-apigateway-authorizer:
+                  type: "token"
+                  authorizerUri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:cs-token-auth-wpst-demo-ui/invocations"
+                  authorizerResultTtlInSeconds: 0
+            definitions:
+              Empty:
+                type: "object"
+                title: "Empty Schema"
+            x-amazon-apigateway-request-validators:
+              Validate body, query string parameters, and headers:
+                validateRequestParameters: true
+                validateRequestBody: true
+
     in: "header"
     x-amazon-apigateway-authtype: "custom"
     x-amazon-apigateway-authorizer:

etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa.yaml

@@ -16,7 +16,7 @@
 
 setup(
     name="cumulus_lambda_functions",
-    version="1.2.3",
+    version="1.3.3",
     packages=find_packages(),
     install_requires=install_requires,
     tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage'],

setup.py

@@ -68,6 +68,28 @@ resource "aws_lambda_function" "cumulus_granules_dapa" {
     }
   }
 
+  vpc_config {
+    subnet_ids         = var.cumulus_lambda_subnet_ids
+    security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id]
+  }
+  tags = var.tags
+}
+
+resource "aws_lambda_function" "cumulus_granules_dapa" {
+  filename      = local.lambda_file_name
+  function_name = "${var.prefix}-cumulus_collections_dapa"
+  role          = data.aws_iam_role.unity_cumulus_lambda_role.arn
+  handler       = "cumulus_lambda_functions.cumulus_collections_dapa.lambda_function.lambda_handler"
+  runtime       = "python3.7"
+  timeout       = 300
+
+  environment {
+    variables = {
+      CUMULUS_BASE = var.cumulus_base
+      CUMULUS_LAMBDA_PREFIX = var.prefix
+    }
+  }
+
   vpc_config {
     subnet_ids         = var.cumulus_lambda_subnet_ids
     security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id]

tf-module/unity-cumulus/main.tf

@@ -33,3 +33,5 @@ system_bucket = "am-uds-dev-cumulus-internal"
 cumulus_lambda_vpc_id        = "vpc-06e627ef021d1854e"
 security_group_ids = ["sg-045f9c24c760940b6"]
 aws_region = "us-west-2"
+aws_region = "us-west-2"
+cumulus_base = "https://axhmoecy02.execute-api.us-west-2.amazonaws.com/dev"