From c0f1f0c70bf0a4c3293ec2d9a02a663570e4eaa8 Mon Sep 17 00:00:00 2001 From: wphyojpl <38299756+wphyojpl@users.noreply.github.com> Date: Wed, 21 Sep 2022 16:56:31 -0700 Subject: [PATCH] feat: Collection Creation endpoint with DAPA format (#79) * chore: move the unit test * feat: add create collection for cumulus * feat: adding stac to cumulus for json * feat: finished transforming from stac to cumulus * fix: try creating collection with min requirement * feat: add class to create dapa collection obj * fix: add class to create dapa collection object for unity * feat: add class to create STAC collection * feat: update to python3.9 + new lambda for collection creation * chore: python3.9 for github action * fix: ci/cd on develop branch with additional release postfix * fix: testing different output number * fix: testing different output number * fix: testing different output number * fix: testing different output number * fix: testing different output number * fix: testing different output number * fix: testing different output number * fix: testing different output number * fix: testing different output number * chore: add testcase.. in progress * fix: wrong handler for new lambda * fix: require libraries at the worng location * fix: add mock url if url_path is misisng * feat: add rule when creating a collection * fix: wrong quotes in terraform * fix: use thread to run the actual collection creation in background thread * fix: revert background process. does not work as expected in lambda * fix: addig facade for collection creation * fix: api gateway wants string for body. not dict * fix: no need to hash the dict * feat: add single collection details * feat: add single collection detail logic * fix: update testcase + yaml for api gateway * feat: delete collection if rule creation fails * chore: update version --- .github/workflows/dockerbuild.yml | 2 +- .github/workflows/makefile.yml | 14 +- ci.cd/Makefile | 4 + ci.cd/create_aws_lambda_zip.sh | 12 +- .../cumulus_collections_dapa.py | 19 +- .../cumulus_create_collection_dapa.py | 100 ++++++++++ .../lambda_function.py | 6 + .../cumulus_granules_dapa.py | 6 +- .../cumulus_granules_dapa_ingest_cnm.py | 2 +- .../cumulus_stac/collection_transformer.py | 108 +++++++++-- .../cumulus_stac/unity_collection_stac.py | 69 +++++++ .../upload_granules.py | 7 +- .../cumulus_wrapper/query_collections.py | 180 +++++++++++++++++- .../lib/aws/aws_lambda.py | 20 ++ ...wagger-apigateway-am-uds-dapa-mcp-dev.yaml | 170 +++++++++++++++++ ...agger-apigateway-am-uds-dapa-mcp-test.yaml | 170 +++++++++++++++++ ...ay-dev-swagger-apigateway-am-uds-dapa.yaml | 79 +++++++- setup.py | 3 +- .../cumulus_collections_dapa/__init__.py | 0 .../test_cumulus_create_collection_dapa.py | 83 ++++++++ .../test_collection_transformer.py | 16 +- .../test_unity_collection_stac.py | 26 +++ .../cumulus_wrapper/__init__.py | 0 .../cumulus_wrapper/test_query_collection.py | 180 ++++++++++++++++++ .../cumulus_wrapper/test_query_granules.py | 16 ++ .../lib/aws/__init__.py | 0 .../lib/aws/test_aws_lambda.py | 31 +++ tests/integration_tests/test_dapa_stac.py | 4 +- tf-module/unity-cumulus/main.tf | 59 +++++- tf-module/unity-cumulus/variables.tf | 5 + 30 files changed, 1338 insertions(+), 53 deletions(-) create mode 100644 cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py create mode 100644 cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py create mode 100644 cumulus_lambda_functions/lib/aws/aws_lambda.py create mode 100644 etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-dev.yaml create mode 100644 etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-test.yaml create mode 100644 tests/cumulus_lambda_functions/cumulus_collections_dapa/__init__.py create mode 100644 tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py create mode 100644 tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py create mode 100644 tests/cumulus_lambda_functions/cumulus_wrapper/__init__.py create mode 100644 tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py create mode 100644 tests/cumulus_lambda_functions/cumulus_wrapper/test_query_granules.py create mode 100644 tests/cumulus_lambda_functions/lib/aws/__init__.py create mode 100644 tests/cumulus_lambda_functions/lib/aws/test_aws_lambda.py diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml index fabbff99..dd824895 100644 --- a/.github/workflows/dockerbuild.yml +++ b/.github/workflows/dockerbuild.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.7' + python-version: '3.9' - run: | # make file runnable, might not be necessary chmod +x "${GITHUB_WORKSPACE}/ci.cd/store_version.sh" diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index f08a2c48..72e24062 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -2,9 +2,9 @@ name: Makefile CI on: push: - branches: [ main ] -# pull_request: -# branches: [ main ] + branches: [ main, develop ] + pull_request: + branches: [ develop ] env: ARTIFACT_BASE_NAME: cumulus_lambda_functions @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.7' + python-version: '3.9' - run: | python3 "${GITHUB_WORKSPACE}/setup.py" install - run: | @@ -52,7 +52,7 @@ jobs: prerelease: false - name: Create PreRelease id: create_prerelease - if: ${{ contains(github.ref, 'main') }} +# if: ${{ contains(github.ref, 'main') }} uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token @@ -66,7 +66,7 @@ jobs: prerelease: true - name: Upload PreRelease Asset 1 id: upload-prerelease-asset-1 - if: ${{ contains(github.ref, 'main') }} +# if: ${{ contains(github.ref, 'main') }} uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -77,7 +77,7 @@ jobs: asset_content_type: application/zip - name: Upload PreRelease Asset 2 id: upload-prerelease-asset-2 - if: ${{ contains(github.ref, 'main') }} +# if: ${{ contains(github.ref, 'main') }} uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci.cd/Makefile b/ci.cd/Makefile index f9ff7556..ff57e709 100644 --- a/ci.cd/Makefile +++ b/ci.cd/Makefile @@ -25,3 +25,7 @@ update_lambda_function_2: aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_granules_dapa --publish &>/dev/null update_lambda_function_3: aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa --publish &>/dev/null +update_lambda_function_4: + aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa --publish &>/dev/null +update_lambda_function_5: + aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null diff --git a/ci.cd/create_aws_lambda_zip.sh b/ci.cd/create_aws_lambda_zip.sh index 2a3ef1ea..51052696 100644 --- a/ci.cd/create_aws_lambda_zip.sh +++ b/ci.cd/create_aws_lambda_zip.sh @@ -22,6 +22,16 @@ cp ${zip_file} build/ cd $project_root_dir/tf-module/unity-cumulus zip -9 ${terraform_zip_file} * **/* +# github.job +github_branch=${GITHUB_REF##*/} +software_version_trailing="" +main_branch="main" +if [ "$github_branch" = "$main_branch" ]; +then + software_version="" +else + software_version_trailing="-${github_branch}-${GITHUB_RUN_ID}" +fi software_version=`python3 ${project_root_dir}/setup.py --version` -echo "software_version=${software_version}" >> ${GITHUB_ENV} +echo "software_version=${software_version}${software_version_trailing}" >> ${GITHUB_ENV} cat ${GITHUB_ENV} diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py index 4028a96f..5f3fcec9 100644 --- a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py +++ b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py @@ -28,6 +28,19 @@ def __init__(self, event): self.__cumulus = CollectionsQuery(self.__cumulus_base, self.__jwt_token) self.__cumulus.with_limit(self.__limit) self.__cumulus.with_page_number(self.__page_number) + self.__get_collection_id() + + def __get_collection_id(self): + if 'pathParameters' not in self.__event: + return self + path_param_dict = self.__event['pathParameters'] + if 'collectionId' not in path_param_dict: + return self + collection_id = path_param_dict['collectionId'] + if collection_id == '*': + return self + self.__cumulus.with_collection_id(path_param_dict['collectionId']) + return self def __assign_values(self): if 'queryStringParameters' not in self.__event or self.__event['queryStringParameters'] is None: @@ -61,12 +74,12 @@ def start(self): if 'server_error' in cumulus_result: return { 'statusCode': 500, - 'body': {'message': cumulus_result['server_error']} + 'body': json.dumps({'message': cumulus_result['server_error']}) } if 'client_error' in cumulus_result: return { 'statusCode': 400, - 'body': {'message': cumulus_result['client_error']} + 'body': json.dumps({'message': cumulus_result['client_error']}) } cumulus_size = self.__get_size() return { @@ -84,5 +97,5 @@ def start(self): LOGGER.exception(f'unexpected error') return { 'statusCode': 500, - 'body': {'message': f'unpredicted error: {str(e)}'} + 'body': json.dumps({'message': f'unpredicted error: {str(e)}'}) } diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py new file mode 100644 index 00000000..abbdbecb --- /dev/null +++ b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py @@ -0,0 +1,100 @@ +import json +import os + +import pystac + +from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer +from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery +from cumulus_lambda_functions.lib.aws.aws_lambda import AwsLambda +from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator + +LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) + + +class CumulusCreateCollectionDapa: + def __init__(self, event): + required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL'] + if not all([k in os.environ for k in required_env]): + raise EnvironmentError(f'one or more missing env: {required_env}') + self.__event = event + self.__request_body = None + self.__cumulus_collection_query = CollectionsQuery('', '') + self.__cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX') + self.__ingest_sqs_url = os.getenv('CUMULUS_WORKFLOW_SQS_URL') + self.__workflow_name = os.getenv('CUMULUS_WORKFLOW_NAME', 'CatalogGranule') + self.__provider_id = '' # TODO. need this? + self.__collection_creation_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip() + + def execute_creation(self): + try: + cumulus_collection_doc = CollectionTransformer().from_stac(self.__request_body) + creation_result = self.__cumulus_collection_query.create_collection(cumulus_collection_doc, self.__cumulus_lambda_prefix) + if 'status' not in creation_result: + LOGGER.error(f'status not in creation_result: {creation_result}') + return { + 'statusCode': 500, + 'body': json.dumps({ + 'message': creation_result + }) + } + rule_creation_result = self.__cumulus_collection_query.create_sqs_rules( + cumulus_collection_doc, + self.__cumulus_lambda_prefix, + self.__ingest_sqs_url, + self.__provider_id, + self.__workflow_name, + ) + if 'status' not in rule_creation_result: + LOGGER.error(f'status not in rule_creation_result. deleting collection: {rule_creation_result}') + delete_collection_result = self.__cumulus_collection_query.delete_collection(self.__cumulus_lambda_prefix, cumulus_collection_doc['name'], cumulus_collection_doc['version']) + return { + 'statusCode': 500, + 'body': json.dumps({ + 'message': {rule_creation_result}, + 'details': f'collection deletion result: {delete_collection_result}' + }) + } + except Exception as e: + LOGGER.exception('error while creating new collection in Cumulus') + return { + 'statusCode': 500, + 'body': json.dumps({ + 'message': f'error while creating new collection in Cumulus. check details', + 'details': str(e) + }) + } + LOGGER.info(f'creation_result: {creation_result}') + return { + 'statusCode': 200, + 'body': json.dumps({ + 'message': creation_result + }) + } + + def start(self): + if 'body' not in self.__event: + raise ValueError(f'missing body in {self.__event}') + self.__request_body = json.loads(self.__event['body']) + LOGGER.debug(f'request body: {self.__request_body}') + validation_result = pystac.Collection.from_dict(self.__request_body).validate() + if not isinstance(validation_result, list): + LOGGER.error(f'request body is not valid STAC collection: {validation_result}') + return { + 'statusCode': 500, + 'body': json.dumps({'message': f'request body is not valid STAC Collection schema. check details', + 'details': validation_result}) + } + if self.__collection_creation_lambda_name != '': + response = AwsLambda().invoke_function( + function_name=self.__collection_creation_lambda_name, + payload=self.__event, + ) + LOGGER.debug(f'async function started: {response}') + return { + 'statusCode': 202, + 'body': json.dumps({ + 'message': 'processing' + }) + } + LOGGER.debug(f'creating collection.') + return self.execute_creation() diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py b/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py index 459d4c7d..598fb32b 100644 --- a/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py +++ b/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py @@ -1,4 +1,5 @@ from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_collections_dapa import CumulusCollectionsDapa +from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_create_collection_dapa import CumulusCreateCollectionDapa from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator @@ -12,3 +13,8 @@ def lambda_handler(event, context): LambdaLoggerGenerator.remove_default_handlers() # TODO implement return CumulusCollectionsDapa(event).start() + + +def lambda_handler_ingestion(event, context): + LambdaLoggerGenerator.remove_default_handlers() + return CumulusCreateCollectionDapa(event).start() diff --git a/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py b/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py index 124af005..ca13b8cc 100644 --- a/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py +++ b/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py @@ -110,12 +110,12 @@ def start(self): if 'server_error' in cumulus_result: return { 'statusCode': 500, - 'body': {'message': cumulus_result['server_error']} + 'body': json.dumps({'message': cumulus_result['server_error']}) } if 'client_error' in cumulus_result: return { 'statusCode': 400, - 'body': {'message': cumulus_result['client_error']} + 'body': json.dumps({'message': cumulus_result['client_error']}) } cumulus_size = self.__get_size() return { @@ -133,5 +133,5 @@ def start(self): LOGGER.exception(f'unexpected error') return { 'statusCode': 500, - 'body': {'message': f'unpredicted error: {str(e)}'} + 'body': json.dumps({'message': f'unpredicted error: {str(e)}'}) } diff --git a/cumulus_lambda_functions/cumulus_granules_dapa_ingest_cnm/cumulus_granules_dapa_ingest_cnm.py b/cumulus_lambda_functions/cumulus_granules_dapa_ingest_cnm/cumulus_granules_dapa_ingest_cnm.py index 7e49693d..d2c9469e 100644 --- a/cumulus_lambda_functions/cumulus_granules_dapa_ingest_cnm/cumulus_granules_dapa_ingest_cnm.py +++ b/cumulus_lambda_functions/cumulus_granules_dapa_ingest_cnm/cumulus_granules_dapa_ingest_cnm.py @@ -193,5 +193,5 @@ def start(self): } return { 'statusCode': 500, - 'body': {'message': f'failed {len(error_list)}/{len(self.__request_body["features"])}', 'details': error_list} + 'body': json.dumps({'message': f'failed {len(error_list)}/{len(self.__request_body["features"])}', 'details': error_list}) } diff --git a/cumulus_lambda_functions/cumulus_stac/collection_transformer.py b/cumulus_lambda_functions/cumulus_stac/collection_transformer.py index fd608660..a641da81 100644 --- a/cumulus_lambda_functions/cumulus_stac/collection_transformer.py +++ b/cumulus_lambda_functions/cumulus_stac/collection_transformer.py @@ -1,9 +1,12 @@ import json from datetime import datetime -from urllib.parse import quote_plus +from urllib.parse import quote_plus, urlparse, unquote_plus +import pystac +from pystac import Link from cumulus_lambda_functions.cumulus_stac.stac_transformer_abstract import StacTransformerAbstract +from cumulus_lambda_functions.lib.time_utils import TimeUtils STAC_COLLECTION_SCHEMA = '''{ "$schema": "http://json-schema.org/draft-07/schema#", @@ -281,11 +284,21 @@ class CollectionTransformer(StacTransformerAbstract): - def __init__(self): + def __init__(self, report_to_ems:bool = True, include_date_range=False): self.__stac_collection_schema = json.loads(STAC_COLLECTION_SCHEMA) self.__cumulus_collection_schema = {} + self.__report_to_ems = report_to_ems + self.__include_date_range = include_date_range - def __convert_to_stac_links(self, collection_file_obj: dict): + def generate_target_link_url(self, regex: str = None, bucket: str = None): + href_link = ['unknown_bucket', 'unknown_regex'] + if regex is not None and regex != '': + href_link[1] = regex + if bucket is not None and bucket != '': + href_link[0] = bucket + return f"./collection.json?bucket={href_link[0]}®ex={quote_plus(href_link[1])}" + + def __convert_to_stac_links(self, collection_file_obj: dict, rel_type: str = 'item'): """ expected output { @@ -310,18 +323,16 @@ def __convert_to_stac_links(self, collection_file_obj: dict): if collection_file_obj is None: return {} stac_link = { - 'rel': 'item', + 'rel': rel_type, } if 'type' in collection_file_obj: stac_link['type'] = collection_file_obj['type'] if 'sampleFileName' in collection_file_obj: stac_link['title'] = collection_file_obj['sampleFileName'] - href_link = ['unknown_bucket', 'unknown_regex'] - if 'bucket' in collection_file_obj: - href_link[0] = collection_file_obj['bucket'] - if 'regex' in collection_file_obj: - href_link[1] = collection_file_obj['regex'] - stac_link['href'] = f"./collection.json?bucket={href_link[0]}®ex={quote_plus(href_link[1])}" + stac_link['href'] = self.generate_target_link_url( + collection_file_obj['regex'] if 'regex' in collection_file_obj else None, + collection_file_obj['bucket'] if 'bucket' in collection_file_obj else None, + ) return stac_link # def to_pystac_link_obj(self, input_dict: dict): @@ -418,14 +429,77 @@ def to_stac(self, source: dict) -> dict: "process": [source['process'] if 'process' in source else ''], "totalGranules": [source['total_size'] if 'total_size' in source else -1], }, - "links": [{ - "rel": "root", - "type": "application/json", - "title": f"{source['name']}___{source['version']}", - "href": "./collection.json" - }] + [self.__convert_to_stac_links(k) for k in source['files']], + "links": [self.__convert_to_stac_links({ + "regex": source['url_path'] if 'url_path' in source else './collection.json', + "sampleFileName": source['sampleFileName'], + "type": "application/json", + + }, 'root')] + [self.__convert_to_stac_links(k) for k in source['files']], } return stac_collection + def get_href(self, input_href: str): + parse_result = urlparse(input_href) + if parse_result.query == '': + return '' + query_dict = [k.split('=') for k in parse_result.query.split('&')] + query_dict = {k[0]: unquote_plus(k[1]) for k in query_dict} + return query_dict + + def __convert_from_stac_links(self, link_obj: dict): + output_file_object = { + 'reportToEms': self.__report_to_ems + } + if 'type' in link_obj: + output_file_object['type'] = link_obj['type'] + if 'title' in link_obj: + output_file_object['sampleFileName'] = link_obj['title'] + if 'href' in link_obj: + href_dict = self.get_href(link_obj['href']) + if 'bucket' in href_dict: + output_file_object['bucket'] = href_dict['bucket'] + if 'regex' in href_dict: + output_file_object['regex'] = href_dict['regex'] + return output_file_object + def from_stac(self, source: dict) -> dict: - return {} + input_dapa_collection = pystac.Collection.from_dict(source) + if not input_dapa_collection.validate(): + raise ValueError(f'invalid source dapa: {input_dapa_collection}') + output_collection_cumulus = { + # "createdAt": 1647992847582, + "reportToEms": self.__report_to_ems, + "duplicateHandling": "skip", + # "updatedAt": 1647992847582, + # "timestamp": 1647992849273 + } + summaries = input_dapa_collection.summaries.lists + if 'granuleId' in summaries: + output_collection_cumulus['granuleId'] = summaries['granuleId'][0] + if 'granuleIdExtraction' in summaries: + output_collection_cumulus['granuleIdExtraction'] = summaries['granuleIdExtraction'][0] + if 'process' in summaries: + output_collection_cumulus['process'] = summaries['process'][0] + name_version = input_dapa_collection.id.split('___') + output_collection_cumulus['name'] = name_version[0] + output_collection_cumulus['version'] = name_version[1] + output_files = [] + for each_link_obj in input_dapa_collection.links: + each_link_obj: Link = each_link_obj + each_file_obj = self.__convert_from_stac_links(each_link_obj.to_dict()) + if each_link_obj.rel == 'root': + if 'regex' in each_file_obj: + output_collection_cumulus['url_path'] = each_file_obj['regex'] + if 'sampleFileName' in each_file_obj: + output_collection_cumulus['sampleFileName'] = each_file_obj['sampleFileName'] + else: + output_files.append(each_file_obj) + output_collection_cumulus['files'] = output_files + if len(input_dapa_collection.extent.temporal.intervals) > 0: + date_interval = input_dapa_collection.extent.temporal.intervals[0] + if len(date_interval) == 2 and self.__include_date_range is True: + if date_interval[0] is not None: + output_collection_cumulus['dateFrom'] = date_interval[0].strftime(TimeUtils.MMDD_FORMAT) + if date_interval[1] is not None: + output_collection_cumulus['dateTo'] = date_interval[1].strftime(TimeUtils.MMDD_FORMAT) + return output_collection_cumulus diff --git a/cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py b/cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py new file mode 100644 index 00000000..df1245bd --- /dev/null +++ b/cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py @@ -0,0 +1,69 @@ +from datetime import datetime + +from pystac import Link, Collection, Extent, SpatialExtent, TemporalExtent, Summaries + +from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer +from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator + +LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) + + +class UnityCollectionStac: + def __init__(self): + self.__id = '' + self.__granule_id_extraction_regex = '' + self.__process = '' + self.__collection_title = '' + self.__granule_id_regex = '' + self.__sample_filename = '' + self.__files = [] + self.__collection_transformer = CollectionTransformer() + + def with_title(self, title: str): + self.__collection_title = title + return self + + def with_process(self, process: str): + self.__process = process + return self + + def with_id(self, collection_id: str): + self.__id = collection_id + if '___' not in collection_id: + LOGGER.warning(f'no ID in {collection_id}. using 001') + self.__id = f'{self.__id}___001' + return self + + def with_graule_id_regex(self, granule_id_regex): + self.__granule_id_regex = granule_id_regex + return self + + def with_granule_id_extraction_regex(self, granule_id_extraction_regex): + self.__granule_id_extraction_regex = granule_id_extraction_regex + return self + + def add_file_type(self, title: str, regex: str, bucket: str, media_type: str, rel: str = 'item'): + if rel == 'root': + LOGGER.debug('updating media_type for rel = root') + media_type = 'application/json' + self.__files.append(Link(rel=rel, target=self.__collection_transformer.generate_target_link_url(regex, bucket), media_type=media_type, title=title)) + return self + + def start(self): + # TODO validate + stac_collection = Collection(id=self.__id, + description='TODO', + extent=Extent(SpatialExtent([[0, 0, 0, 0]]), + TemporalExtent([[datetime.utcnow(), datetime.utcnow()]])), + title=self.__collection_title, + summaries=Summaries({ + 'granuleId': [self.__granule_id_regex], + 'granuleIdExtraction': [self.__granule_id_extraction_regex], + 'process': [self.__process] + }), + ) + stac_collection.add_links(self.__files) + new_collection = stac_collection.to_dict(include_self_link=False) + if 'links' in new_collection and len(new_collection['links']) > 0 and new_collection['links'][0]['rel'] == 'root': + new_collection['links'][0]['href'] = './collection.json' + return new_collection diff --git a/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py b/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py index 1c5a79f1..7d22c649 100644 --- a/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py +++ b/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse, unquote_plus from cumulus_lambda_functions.cumulus_dapa_client.dapa_client import DapaClient +from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3 LOGGER = logging.getLogger(__name__) @@ -49,11 +50,7 @@ def __set_props_from_env(self): return self def __get_href(self, input_href: str): - parse_result = urlparse(input_href) - if parse_result.query == '': - return '' - query_dict = [k.split('=') for k in parse_result.query.split('&')] - query_dict = {k[0]: unquote_plus(k[1]) for k in query_dict} + query_dict = CollectionTransformer().get_href(input_href) if 'regex' not in query_dict: raise ValueError(f'missing regex in {input_href}') return query_dict['regex'] diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_collections.py b/cumulus_lambda_functions/cumulus_wrapper/query_collections.py index 8788b52c..13a00863 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_collections.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_collections.py @@ -11,17 +11,36 @@ class CollectionsQuery(CumulusBase): __collections_key = 'collections' + __rules_key = 'rules' __stats_key = 'stats' + __collection_id_key = 'collectionId' + __collection_name = 'name' + __collection_version = 'version' + def __init__(self, cumulus_base: str, cumulus_token: str): super().__init__(cumulus_base, cumulus_token) + def with_collection_id(self, collection_id: str): + # self._conditions.append(f'{self.__collection_id_key}={collection_id}') + split_collection = collection_id.split('___') + self._conditions.append(f'{self.__collection_name}={split_collection[0]}') + self._conditions.append(f'{self.__collection_version}={split_collection[1]}') + + return self + def get_size(self, private_api_prefix: str): + query_params = {'field': 'status', 'type': 'collections'} + main_conditions = {k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]} + if self.__collection_name in main_conditions: + query_params[self.__collection_name] = main_conditions[self.__collection_name] + if self.__collection_version in main_conditions: + query_params[self.__collection_version] = main_conditions[self.__collection_version] payload = { 'httpMethod': 'GET', 'resource': '/{proxy+}', 'path': f'/stats/aggregate', - 'queryStringParameters': {'field': 'status', 'type': 'collections'}, + 'queryStringParameters': query_params, 'headers': { 'Content-Type': 'application/json', }, @@ -46,6 +65,165 @@ def get_size(self, private_api_prefix: str): total_size = query_result['meta']['count'] return {'total_size': total_size} + def create_collection(self, new_collection: dict, private_api_prefix: str): + payload = { + 'httpMethod': 'POST', + 'resource': '/{proxy+}', + 'path': f'/{self.__collections_key}', + 'headers': { + 'Content-Type': 'application/json', + }, + 'body': json.dumps(new_collection) + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 500, 'body': '', 'headers': {}} + """ + if query_result['statusCode'] >= 500: + LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}') + return {'server_error': query_result} + if query_result['statusCode'] >= 400: + LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}') + return {'client_error': query_result} + query_result = json.loads(query_result['body']) + LOGGER.debug(f'json query_result: {query_result}') + if 'message' not in query_result: + return {'server_error': f'invalid response: {query_result}'} + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + return {'status': query_result['message']} + + def delete_collection(self, private_api_prefix, collection_id, collection_version): + payload = { + 'httpMethod': 'DELETE', + 'resource': '/{proxy+}', + 'path': f'/{self.__collections_key}/{collection_id}/{collection_version}', + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 500, 'body': '', 'headers': {}} + """ + if query_result['statusCode'] >= 500: + LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}') + return {'server_error': query_result} + if query_result['statusCode'] >= 400: + LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}') + return {'client_error': query_result} + query_result = json.loads(query_result['body']) + LOGGER.debug(f'json query_result: {query_result}') + if 'message' not in query_result: + return {'server_error': f'invalid response: {query_result}'} + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + return {'status': query_result['message']} + + def query_rules(self, private_api_prefix: str): + payload = { + 'httpMethod': 'GET', + 'resource': '/{proxy+}', + 'path': f'/{self.__rules_key}', + # 'queryStringParameters': {k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]}, + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False} + """ + if query_result['statusCode'] >= 500: + LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}') + return {'server_error': query_result} + if query_result['statusCode'] >= 400: + LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}') + return {'client_error': query_result} + query_result = json.loads(query_result['body']) + LOGGER.debug(f'json query_result: {query_result}') + if 'results' not in query_result: + LOGGER.error(f'missing key: results. invalid response json: {query_result}') + return {'server_error': f'missing key: results. invalid response json: {query_result}'} + query_result = query_result['results'] + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + return {'results': query_result} + + def create_sqs_rules(self, new_collection: dict, private_api_prefix: str, sqs_url: str, provider_name: str = '', workflow_name: str = 'CatalogGranule', visibility_timeout: int = 1800): + """ +curl --request POST "$CUMULUS_BASEURL/rules" --header "Authorization: Bearer $cumulus_token" --header 'Content-Type: application/json' --data '{ + "workflow": "DiscoverGranules", + "collection": { + "name": "ATMS_SCIENCE_Group", + "version": "001" + }, + "provider": "snpp_provider_03", + "name": "ATMS_SCIENCE_Group_2016_002_v1", + "rule": { + "type": "onetime" + }, + "meta": { "provider_path": "data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2016/002/", "publish": false, "distribution_endpoint": "s3://am-uds-dev-cumulus-internal/" }, + "state": "ENABLED" +}' + :return: + """ + payload = { + 'httpMethod': 'POST', + 'resource': '/{proxy+}', + 'path': f'/{self.__rules_key}', + 'headers': { + 'Content-Type': 'application/json', + }, + 'body': json.dumps({ + 'workflow': workflow_name, + 'collection': { + 'name': new_collection['name'], + 'version': new_collection['version'], + }, + # 'provider': provider_name, + 'name': f'{new_collection["name"]}___{new_collection["version"]}___rules_sqs', + 'rule': { + # 'type': 'onetime', + 'type': 'sqs', + 'value': sqs_url, + }, + 'state': 'ENABLED', + "meta": { + 'retries': 1, + 'visibilityTimeout': visibility_timeout, + # "provider_path": "data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2016/002/", + # "publish": False, + # "distribution_endpoint": "s3://am-uds-dev-cumulus-internal/" + }, + + }) + } + if provider_name is not None and provider_name != '': + payload['provider'] = provider_name + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 500, 'body': '', 'headers': {}} + """ + if query_result['statusCode'] >= 500: + LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}') + return {'server_error': query_result} + if query_result['statusCode'] >= 400: + LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}') + return {'client_error': query_result} + query_result = json.loads(query_result['body']) + LOGGER.debug(f'json query_result: {query_result}') + if 'message' not in query_result: + return {'server_error': f'invalid response: {query_result}'} + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + return {'status': query_result['message']} def __get_stats(self, collection_id, private_api_prefix: str): payload = { diff --git a/cumulus_lambda_functions/lib/aws/aws_lambda.py b/cumulus_lambda_functions/lib/aws/aws_lambda.py new file mode 100644 index 00000000..b9e168bc --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/aws_lambda.py @@ -0,0 +1,20 @@ +import json + +from cumulus_lambda_functions.lib.aws.aws_cred import AwsCred + + +class AwsLambda(AwsCred): + def __init__(self): + super().__init__() + self.__lambda_client = self.get_client('lambda') + + def invoke_function(self, function_name: str, payload: dict): + response = self.__lambda_client.invoke( + FunctionName=function_name, + InvocationType='Event', # 'Event' = async | 'RequestResponse' = sync | 'DryRun', + LogType='None', # 'None' = async | 'Tail = sync', + ClientContext='', # Up to 3583 bytes of base64-encoded data + Payload=json.dumps(payload).encode(), + # Qualifier='string' + ) + return response diff --git a/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-dev.yaml b/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-dev.yaml new file mode 100644 index 00000000..92bf7dc9 --- /dev/null +++ b/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-dev.yaml @@ -0,0 +1,170 @@ +--- +swagger: "2.0" +info: + description: "Unity API Gateway" + version: "2022-05-04T20:23:33Z" + title: "Unity API Gateway" +host: " 1gp9st60gd.execute-api.us-west-2.amazonaws.com" +basePath: "/dev" +schemes: +- "https" +paths: + /am-uds-dapa/collections: + get: + produces: + - "application/json" + parameters: + - name: "limit" + in: "query" + required: false + type: "string" + - name: "offset" + in: "query" + required: false + type: "string" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + put: + produces: + - "application/json" + body: + schema: + $ref: "#/definitions/Empty" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + post: + produces: + - "application/json" + body: + schema: + $ref: "#/definitions/Empty" + responses: + "202": + description: "202 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_creation_dapa_facade/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + /am-uds-dapa/collections/{collectionId}: + get: + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_collections_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + + /am-uds-dapa/collections/{collectionId}/items: + get: + produces: + - "application/json" + parameters: + - name: "datetime" + in: "query" + required: false + type: "string" + - name: "limit" + in: "query" + required: false + type: "string" + - name: "offset" + in: "query" + required: false + type: "string" + - name: "bbox" + in: "query" + required: false + type: "string" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:237868187491:function:uds-dev-cumulus-cumulus_granules_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" +securityDefinitions: + Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: + type: "apiKey" + name: "Authorization" + in: "header" + x-amazon-apigateway-authtype: "custom" + x-amazon-apigateway-authorizer: + type: "token" + authorizerUri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:237868187491:function:cs-lambda-authorizer/invocations" + authorizerResultTtlInSeconds: 0 +definitions: + Empty: + type: "object" + title: "Empty Schema" +x-amazon-apigateway-request-validators: + Validate body, query string parameters, and headers: + validateRequestParameters: true + validateRequestBody: true diff --git a/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-test.yaml b/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-test.yaml new file mode 100644 index 00000000..9ae95f8b --- /dev/null +++ b/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa-mcp-test.yaml @@ -0,0 +1,170 @@ +--- +swagger: "2.0" +info: + description: "Unity API Gateway" + version: "2022-05-04T20:23:33Z" + title: "Unity API Gateway" +host: "58nbcawrvb.execute-api.us-west-2.amazonaws.com" +basePath: "/dev" +schemes: +- "https" +paths: + /am-uds-dapa/collections: + get: + produces: + - "application/json" + parameters: + - name: "limit" + in: "query" + required: false + type: "string" + - name: "offset" + in: "query" + required: false + type: "string" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:428002334974:function:uds-test-cumulus-cumulus_collections_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + put: + produces: + - "application/json" + body: + schema: + $ref: "#/definitions/Empty" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:428002334974:function:uds-test-cumulus-cumulus_collections_ingest_cnm_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + post: + produces: + - "application/json" + body: + schema: + $ref: "#/definitions/Empty" + responses: + "202": + description: "202 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:428002334974:function:uds-test-cumulus-cumulus_collections_creation_dapa_facade/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + /am-uds-dapa/collections/{collectionId}: + get: + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:428002334974:function:uds-test-cumulus-cumulus_collections_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + + /am-uds-dapa/collections/{collectionId}/items: + get: + produces: + - "application/json" + parameters: + - name: "datetime" + in: "query" + required: false + type: "string" + - name: "limit" + in: "query" + required: false + type: "string" + - name: "offset" + in: "query" + required: false + type: "string" + - name: "bbox" + in: "query" + required: false + type: "string" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:428002334974:function:uds-test-cumulus-cumulus_granules_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" +securityDefinitions: + Unity_API_Gateway_Lambda_HySDS_Authorizer_Node1: + type: "apiKey" + name: "Authorization" + in: "header" + x-amazon-apigateway-authtype: "custom" + x-amazon-apigateway-authorizer: + type: "token" + authorizerUri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:428002334974:function:cs-lambda-authorizer/invocations" + authorizerResultTtlInSeconds: 0 +definitions: + Empty: + type: "object" + title: "Empty Schema" +x-amazon-apigateway-request-validators: + Validate body, query string parameters, and headers: + validateRequestParameters: true + validateRequestBody: true diff --git a/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa.yaml b/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa.yaml index 9341a49b..069d1a74 100644 --- a/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa.yaml +++ b/etc/Unity-API-Gateway-dev-swagger-apigateway-am-uds-dapa.yaml @@ -27,6 +27,8 @@ paths: description: "200 response" schema: $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node: [] x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ \ and headers" x-amazon-apigateway-integration: @@ -38,6 +40,77 @@ paths: statusCode: "200" passthroughBehavior: "when_no_match" contentHandling: "CONVERT_TO_TEXT" + put: + produces: + - "application/json" + body: + schema: + $ref: "#/definitions/Empty" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + post: + produces: + - "application/json" + body: + schema: + $ref: "#/definitions/Empty" + responses: + "202": + description: "202 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa_facade/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + /am-uds-dapa/collections/{collectionId}: + get: + produces: + - "application/json" + responses: + "200": + description: "200 response" + schema: + $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node: [] + x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ + \ and headers" + x-amazon-apigateway-integration: + type: "aws_proxy" + httpMethod: "POST" + uri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_dapa/invocations" + responses: + default: + statusCode: "200" + passthroughBehavior: "when_no_match" + contentHandling: "CONVERT_TO_TEXT" + /am-uds-dapa/collections/{collectionId}/items: get: produces: @@ -64,6 +137,8 @@ paths: description: "200 response" schema: $ref: "#/definitions/Empty" + security: + - Unity_API_Gateway_Lambda_HySDS_Authorizer_Node: [] x-amazon-apigateway-request-validator: "Validate body, query string parameters,\ \ and headers" x-amazon-apigateway-integration: @@ -76,14 +151,14 @@ paths: passthroughBehavior: "when_no_match" contentHandling: "CONVERT_TO_TEXT" securityDefinitions: - Unity_API_Gateway_Lambda_WPST_UI_Demo_Authorizer_Node: + Unity_API_Gateway_Lambda_HySDS_Authorizer_Node: type: "apiKey" name: "Authorization" in: "header" x-amazon-apigateway-authtype: "custom" x-amazon-apigateway-authorizer: type: "token" - authorizerUri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:cs-token-auth-wpst-demo-ui/invocations" + authorizerUri: "arn:aws:apigateway:us-west-2:lambda:path/2015-03-31/functions/arn:aws:lambda:us-west-2:884500545225:function:cs-basic-token-auth-node/invocations" authorizerResultTtlInSeconds: 0 definitions: Empty: diff --git a/setup.py b/setup.py index ada5515b..e39db3f2 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ from setuptools import find_packages, setup install_requires = [ + 'pystac', 'jsonschema', 'fastjsonschema', 'xmltodict', 'requests' @@ -16,7 +17,7 @@ setup( name="cumulus_lambda_functions", - version="1.7.2", + version="1.8.0", packages=find_packages(), install_requires=install_requires, tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'], diff --git a/tests/cumulus_lambda_functions/cumulus_collections_dapa/__init__.py b/tests/cumulus_lambda_functions/cumulus_collections_dapa/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py b/tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py new file mode 100644 index 00000000..e20c6223 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py @@ -0,0 +1,83 @@ +import json +import os +from datetime import datetime +from time import sleep +from unittest import TestCase + +import requests + +from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_create_collection_dapa import CumulusCreateCollectionDapa +from cumulus_lambda_functions.cumulus_stac.unity_collection_stac import UnityCollectionStac +from cumulus_lambda_functions.lib.cognito_login.cognito_token_retriever import CognitoTokenRetriever +from cumulus_lambda_functions.lib.constants import Constants + + +class TestCumulusCreateCollectionDapa(TestCase): + def test_01(self): + dapa_collection = UnityCollectionStac() \ + .with_id(f'CUMULUS_DAPA_UNIT_TEST___{int(datetime.utcnow().timestamp())}') \ + .with_graule_id_regex("^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$") \ + .with_granule_id_extraction_regex("(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+") \ + .with_title("P1570515ATMSSCIENCEAXT11344000000001.PDS") \ + .with_process('modis') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + 'internal', 'data', 'item') + os.environ['CUMULUS_LAMBDA_PREFIX'] = 'am-uds-dev-cumulus' + os.environ['CUMULUS_WORKFLOW_SQS_URL'] = 'https://sqs.us-west-2.amazonaws.com/884500545225/am-uds-dev-cumulus-cnm-submission-queue' + stac_collection = dapa_collection.start() + event = { + 'body': json.dumps(stac_collection) + } + creation = CumulusCreateCollectionDapa(event).start() + self.assertTrue('statusCode' in creation, f'missing statusCode: {creation}') + self.assertEqual(200, creation['statusCode'], f'wrong statusCode: {creation}') + return + + def test_02(self): + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + os.environ[Constants.PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.CLIENT_ID] = '7a1fglm2d54eoggj13lccivp25' # JPL Cloud + + os.environ[Constants.COGNITO_URL] = 'https://cognito-idp.us-west-2.amazonaws.com' + bearer_token = CognitoTokenRetriever().start() + post_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev' + post_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/' + headers = { + 'Authorization': f'Bearer {bearer_token}', + # 'Content-Type': 'application/json', + } + temp_collection_id = f'CUMULUS_DAPA_UNIT_TEST___{int(datetime.utcnow().timestamp())}' + dapa_collection = UnityCollectionStac() \ + .with_id(temp_collection_id) \ + .with_graule_id_regex("^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$") \ + .with_granule_id_extraction_regex("(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+") \ + .with_title("P1570515ATMSSCIENCEAXT11344000000001.PDS") \ + .with_process('modis') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + 'internal', 'data', 'item') + stac_collection = dapa_collection.start() + + print(json.dumps(stac_collection)) + query_result = requests.post(url=post_url, + headers=headers, + json=stac_collection, + ) + self.assertEqual(query_result.status_code, 202, f'wrong status code. {query_result.text}') + sleep(60) + collection_created_result = requests.get(url=f'{post_url}{temp_collection_id}', headers=headers) + self.assertEqual(collection_created_result.status_code, 200, f'wrong status code. {collection_created_result.text}') + collection_created_result = json.loads(collection_created_result.text) + self.assertTrue('features' in collection_created_result, f'features not in collection_created_result: {collection_created_result}') + self.assertEqual(len(collection_created_result['features']), 1, f'wrong length: {collection_created_result}') + self.assertEqual(collection_created_result['features'][0]['id'], temp_collection_id, f'wrong id') + # TODO check if collection shows up + return diff --git a/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py b/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py index 9968ea66..af4a67e0 100644 --- a/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py +++ b/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py @@ -51,12 +51,12 @@ def test_01(self): "granuleIdExtraction": "(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+", "reportToEms": True, "version": "001", - "duplicateHandling": "replace", + "duplicateHandling": "skip", "updatedAt": 1647992847582, "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", "timestamp": 1647992849273 } - raw = { + converted_stac = { "type": "Collection", "stac_version": "1.0.0", # "stac_extensions": [], @@ -82,6 +82,14 @@ def test_01(self): }, ] } - raw = CollectionTransformer().to_stac(source) - self.assertEqual(None, stac_validator.validate(raw), f'invalid stac format: {stac_validator}') + converted_stac = CollectionTransformer(include_date_range=True).to_stac(source) + self.assertEqual(None, stac_validator.validate(converted_stac), f'invalid stac format: {stac_validator}') + converted_cumulus = CollectionTransformer(include_date_range=True).from_stac(converted_stac) + for k, v in source.items(): + if k in ['updatedAt', 'timestamp', 'createdAt']: + continue + self.assertTrue(k in converted_cumulus, f'missing {k}') + if k not in ['files', 'dateFrom', 'dateTo']: + self.assertEqual(v, converted_cumulus[k], f'wrong value for {k}') + self.assertEqual(sorted(json.dumps(source['files'])), sorted(json.dumps(converted_cumulus['files'])), f"wrong files content: {source['files']} vs. {converted_cumulus['files']}") return diff --git a/tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py b/tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py new file mode 100644 index 00000000..df970581 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py @@ -0,0 +1,26 @@ +import json +from datetime import datetime +from unittest import TestCase + +import pystac + +from cumulus_lambda_functions.cumulus_stac.unity_collection_stac import \ + UnityCollectionStac + + +class TestUnityCollectionStac(TestCase): + def test_01(self): + dapa_collection = UnityCollectionStac()\ + .with_id(f'CUMULUS_DAPA_UNIT_TEST___{int(datetime.utcnow().timestamp())}')\ + .with_graule_id_regex("^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$")\ + .with_granule_id_extraction_regex("(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+")\ + .with_title("P1570515ATMSSCIENCEAXT11344000000001.PDS")\ + .with_process('modis')\ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", 'internal', 'data', 'item') + + stac_collection = dapa_collection.start() + validation_result = pystac.Collection.from_dict(stac_collection).validate() + self.assertTrue(isinstance(validation_result, list), f'wrong validation for : {json.dumps(stac_collection, indent=4)}. details: {validation_result}') + return diff --git a/tests/cumulus_lambda_functions/cumulus_wrapper/__init__.py b/tests/cumulus_lambda_functions/cumulus_wrapper/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py b/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py new file mode 100644 index 00000000..ba6a0177 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py @@ -0,0 +1,180 @@ +import logging +from datetime import datetime +from unittest import TestCase + +from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery +from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator + + +class TestQueryCollection(TestCase): + def test_01(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_version = int(datetime.utcnow().timestamp()) + sample_collection = { + # "dataType": "MOD09GQ", + # "provider_path": "cumulus-test-data/pdrs", + "name": "UNITY_CUMULUS_DEV_UNIT_TEST", + "version": str(collection_version), + # "process": "modis", + # "duplicateHandling": "skip", + "granuleId": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$", + "granuleIdExtraction": "(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+", + # "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "files": [ + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS", + "type": "data", + "reportToEms": True + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "reportToEms": True, + "type": "metadata" + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "reportToEms": True, + "type": "metadata" + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "reportToEms": True, + "type": "metadata" + } + ], + } + # sample_collection = { + # "createdAt": 1647992847582, + # "reportToEms": True, + # "updatedAt": 1647992847582, + # "timestamp": 1647992849273 + # } + response = collection_query.create_collection(sample_collection, lambda_prefix) + self.assertTrue('status' in response, f'status not in response: {response}') + self.assertEqual('Record saved', response['status'], f'wrong status: {response}') + + delete_response = collection_query.delete_collection(lambda_prefix, 'UNITY_CUMULUS_DEV_UNIT_TEST', str(collection_version)) + self.assertTrue('status' in delete_response, f'status not in response: {response}') + self.assertEqual('Record deleted', delete_response['status'], f'wrong status: {response}') + + return + + def test_02(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_query.with_limit(2) + collections = collection_query.query_direct_to_private_api(lambda_prefix) + self.assertTrue('results' in collections, f'results not in collections: {collections}') + self.assertEqual(2, len(collections['results']), f'wrong length: {collections}') + return + + def test_03(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_query.with_limit(2) + collection_query.with_collection_id('CUMULUS_DAPA_UNIT_TEST___1663627653') + collections = collection_query.query_direct_to_private_api(lambda_prefix) + self.assertTrue('results' in collections, f'results not in collections: {collections}') + self.assertEqual(1, len(collections['results']), f'wrong length: {collections}') + self.assertEqual('CUMULUS_DAPA_UNIT_TEST___1663627653', collections['results'][0]['id'], f'wrong id (DAPA style)') + return + + def test_04(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_query.with_limit(2) + collection_query.with_collection_id('CUMULUS_DAPA_UNIT_TEST___1663627653') + collections = collection_query.get_size(lambda_prefix) + self.assertTrue('total_size' in collections, f'total_size not in collections: {collections}') + self.assertEqual(1, collections['total_size'], f'wrong size: {collections}') + return + + + def test_rules_03(self): + LambdaLoggerGenerator.remove_default_handlers() + # logging.basicConfig(level=20, + # format="%(asctime)s [%(levelname)s] [%(name)s::%(lineno)d] %(message)s") + + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_version = int(datetime.utcnow().timestamp()) + sample_collection = { + # "dataType": "MOD09GQ", + # "provider_path": "cumulus-test-data/pdrs", + "name": "UNITY_CUMULUS_DEV_UNIT_TEST", + "version": str(collection_version), + # "process": "modis", + # "duplicateHandling": "skip", + "granuleId": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$", + "granuleIdExtraction": "(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+", + # "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "files": [ + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS", + "type": "data", + "reportToEms": True + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "reportToEms": True, + "type": "metadata" + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "reportToEms": True, + "type": "metadata" + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "reportToEms": True, + "type": "metadata" + } + ], + } + # sample_collection = { + # "createdAt": 1647992847582, + # "reportToEms": True, + # "updatedAt": 1647992847582, + # "timestamp": 1647992849273 + # } + response = collection_query.create_collection(sample_collection, lambda_prefix) + self.assertTrue('status' in response, f'status not in response: {response}') + self.assertEqual('Record saved', response['status'], f'wrong status: {response}') + + response = collection_query.create_sqs_rules( + sample_collection, + lambda_prefix, + 'https://sqs.us-west-2.amazonaws.com/884500545225/am-uds-dev-cumulus-cnm-submission-queue', + 'SNPP', + 'CatalogGranule' + ) + self.assertTrue('status' in response, f'status not in response: {response}') + self.assertEqual('Record saved', response['status'], f'wrong status: {response}') + return + + def test_rules_04(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_query.with_limit(200) + rules = collection_query.query_rules(lambda_prefix) + self.assertTrue(True) + return diff --git a/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_granules.py b/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_granules.py new file mode 100644 index 00000000..662c4ff1 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_granules.py @@ -0,0 +1,16 @@ +from unittest import TestCase + +from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery + + +class TestGranulesQuery(TestCase): + def test_01(self): + lambda_prefix = 'am-uds-dev-cumulus' + + query_granules = GranulesQuery('NA', 'NA') + query_granules.with_collection_id('SNDR_SNPP_ATMS_L1B_OUTPUT___1') + query_granules.with_limit(7) + granules = query_granules.query_direct_to_private_api(lambda_prefix) + self.assertTrue('results' in granules, f'results not in collections: {granules}') + self.assertEqual(7, len(granules['results']), f'wrong length: {granules}') + return diff --git a/tests/cumulus_lambda_functions/lib/aws/__init__.py b/tests/cumulus_lambda_functions/lib/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/lib/aws/test_aws_lambda.py b/tests/cumulus_lambda_functions/lib/aws/test_aws_lambda.py new file mode 100644 index 00000000..2af27cbe --- /dev/null +++ b/tests/cumulus_lambda_functions/lib/aws/test_aws_lambda.py @@ -0,0 +1,31 @@ +from datetime import datetime +from unittest import TestCase + +from cumulus_lambda_functions.cumulus_stac.unity_collection_stac import UnityCollectionStac +from cumulus_lambda_functions.lib.aws.aws_lambda import AwsLambda + + +class TestAwsLambda(TestCase): + def test_01(self): + dapa_collection = UnityCollectionStac() \ + .with_id(f'CUMULUS_DAPA_UNIT_TEST___{int(datetime.utcnow().timestamp())}') \ + .with_graule_id_regex("^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$") \ + .with_granule_id_extraction_regex("(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+") \ + .with_title("P1570515ATMSSCIENCEAXT11344000000001.PDS") \ + .with_process('modis') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + 'internal', 'data', 'item') + stac_collection = dapa_collection.start() + + response = AwsLambda().invoke_function( + function_name='am-uds-dev-cumulus-cumulus_collections_creation_dapa', + payload=stac_collection, + ) + self.assertTrue('ResponseMetadata' in response, f'missing ResponseMetadata in response {response}') + self.assertTrue('HTTPStatusCode' in response['ResponseMetadata'], f'missing HTTPStatusCode in response#ResponseMetadata {response}') + print(response) + return diff --git a/tests/integration_tests/test_dapa_stac.py b/tests/integration_tests/test_dapa_stac.py index 272d79b5..b51a331e 100644 --- a/tests/integration_tests/test_dapa_stac.py +++ b/tests/integration_tests/test_dapa_stac.py @@ -20,7 +20,7 @@ def test_collection_01(self): .start(base64.standard_b64decode(os.environ.get('USERNAME')).decode(), base64.standard_b64decode(os.environ.get('PASSWORD')).decode()) collection_url = f'{os.environ.get("UNITY_URL")}/am-uds-dapa/collections' response = requests.get(url=collection_url, headers={'Authorization': f'Bearer {cognito_login.token}'}, verify=False) - self.assertEqual(response.status_code, 200, 'wrong status code') + self.assertEqual(response.status_code, 200, f'wrong status code: {response.text}') response_json = json.loads(response.content.decode()) self.assertTrue(len(response_json['features']) > 0, f'empty collections. Need collections to compare') for each_feature in response_json['features']: @@ -37,7 +37,7 @@ def test_granules_01(self): .start(base64.standard_b64decode(os.environ.get('USERNAME')).decode(), base64.standard_b64decode(os.environ.get('PASSWORD')).decode()) collection_url = f'{os.environ.get("UNITY_URL")}/am-uds-dapa/collections/*/items' response = requests.get(url=collection_url, headers={'Authorization': f'Bearer {cognito_login.token}'}, verify=False) - self.assertEqual(response.status_code, 200, 'wrong status code') + self.assertEqual(response.status_code, 200, f'wrong status code: {response.text}') response_json = json.loads(response.content.decode()) self.assertTrue(len(response_json['features']) > 0, f'empty granules. Need collections to compare') for each_feature in response_json['features']: diff --git a/tf-module/unity-cumulus/main.tf b/tf-module/unity-cumulus/main.tf index 33fd759d..f54a8e83 100644 --- a/tf-module/unity-cumulus/main.tf +++ b/tf-module/unity-cumulus/main.tf @@ -38,7 +38,7 @@ resource "aws_lambda_function" "snpp_lvl0_generate_cmr" { function_name = "${var.prefix}-snpp_lvl0_generate_cmr" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.snpp_lvl0_generate_cmr.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { variables = { @@ -58,7 +58,7 @@ resource "aws_lambda_function" "snpp_lvl1_generate_cmr" { function_name = "${var.prefix}-snpp_lvl1_generate_cmr" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.snpp_level1a_generate_cmr.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { variables = { @@ -78,7 +78,7 @@ resource "aws_lambda_function" "cumulus_granules_dapa" { function_name = "${var.prefix}-cumulus_granules_dapa" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.cumulus_granules_dapa.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { @@ -100,7 +100,7 @@ resource "aws_lambda_function" "cumulus_collections_dapa" { function_name = "${var.prefix}-cumulus_collections_dapa" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.cumulus_collections_dapa.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { @@ -122,7 +122,7 @@ resource "aws_lambda_function" "cumulus_collections_ingest_cnm_dapa" { function_name = "${var.prefix}-cumulus_collections_ingest_cnm_dapa" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.cumulus_granules_dapa_ingest_cnm.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { @@ -132,6 +132,55 @@ resource "aws_lambda_function" "cumulus_collections_ingest_cnm_dapa" { } } + vpc_config { + subnet_ids = var.cumulus_lambda_subnet_ids + security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id] + } + tags = var.tags +} + +resource "aws_lambda_function" "cumulus_collections_creation_dapa" { + filename = local.lambda_file_name + function_name = "${var.prefix}-cumulus_collections_creation_dapa" + role = var.lambda_processing_role_arn + handler = "cumulus_lambda_functions.cumulus_collections_dapa.lambda_function.lambda_handler_ingestion" + runtime = "python3.9" + timeout = 300 + + environment { + variables = { + LOG_LEVEL = var.log_level + CUMULUS_LAMBDA_PREFIX = var.prefix + CUMULUS_WORKFLOW_SQS_URL = var.workflow_sqs_url + CUMULUS_WORKFLOW_NAME = "CatalogGranule" + } + } + + vpc_config { + subnet_ids = var.cumulus_lambda_subnet_ids + security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id] + } + tags = var.tags +} + +resource "aws_lambda_function" "cumulus_collections_creation_dapa_facade" { + filename = local.lambda_file_name + function_name = "${var.prefix}-cumulus_collections_creation_dapa_facade" + role = var.lambda_processing_role_arn + handler = "cumulus_lambda_functions.cumulus_collections_dapa.lambda_function.lambda_handler_ingestion" + runtime = "python3.9" + timeout = 300 + + environment { + variables = { + LOG_LEVEL = var.log_level + CUMULUS_LAMBDA_PREFIX = var.prefix + CUMULUS_WORKFLOW_SQS_URL = var.workflow_sqs_url + CUMULUS_WORKFLOW_NAME = "CatalogGranule" + COLLECTION_CREATION_LAMBDA_NAME = aws_lambda_function.cumulus_collections_creation_dapa.arn + } + } + vpc_config { subnet_ids = var.cumulus_lambda_subnet_ids security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id] diff --git a/tf-module/unity-cumulus/variables.tf b/tf-module/unity-cumulus/variables.tf index 98ee669e..ee10f42f 100644 --- a/tf-module/unity-cumulus/variables.tf +++ b/tf-module/unity-cumulus/variables.tf @@ -39,6 +39,11 @@ variable "cnm_sns_topic_arn" { type = string } +variable "workflow_sqs_url" { + type = string + description = "SNS ARN of CNM submission topic" +} + variable "cumulus_base" { type = string description = "Cumulus base URL. Example: https://axhmoecy02.execute-api.us-west-2.amazonaws.com/dev"