diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml index fabbff99..dd824895 100644 --- a/.github/workflows/dockerbuild.yml +++ b/.github/workflows/dockerbuild.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.7' + python-version: '3.9' - run: | # make file runnable, might not be necessary chmod +x "${GITHUB_WORKSPACE}/ci.cd/store_version.sh" diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index f08a2c48..72e24062 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -2,9 +2,9 @@ name: Makefile CI on: push: - branches: [ main ] -# pull_request: -# branches: [ main ] + branches: [ main, develop ] + pull_request: + branches: [ develop ] env: ARTIFACT_BASE_NAME: cumulus_lambda_functions @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: '3.7' + python-version: '3.9' - run: | python3 "${GITHUB_WORKSPACE}/setup.py" install - run: | @@ -52,7 +52,7 @@ jobs: prerelease: false - name: Create PreRelease id: create_prerelease - if: ${{ contains(github.ref, 'main') }} +# if: ${{ contains(github.ref, 'main') }} uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token @@ -66,7 +66,7 @@ jobs: prerelease: true - name: Upload PreRelease Asset 1 id: upload-prerelease-asset-1 - if: ${{ contains(github.ref, 'main') }} +# if: ${{ contains(github.ref, 'main') }} uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -77,7 +77,7 @@ jobs: asset_content_type: application/zip - name: Upload PreRelease Asset 2 id: upload-prerelease-asset-2 - if: ${{ contains(github.ref, 'main') }} +# if: ${{ contains(github.ref, 'main') }} uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci.cd/create_aws_lambda_zip.sh b/ci.cd/create_aws_lambda_zip.sh index 2a3ef1ea..0ddd0e7a 100644 --- a/ci.cd/create_aws_lambda_zip.sh +++ b/ci.cd/create_aws_lambda_zip.sh @@ -22,6 +22,17 @@ cp ${zip_file} build/ cd $project_root_dir/tf-module/unity-cumulus zip -9 ${terraform_zip_file} * **/* +# github.job +github_branch=${GITHUB_REF##*/} +software_version_trailing="" +main_branch="main" +if ["$github_branch"="$main_branch"]; +then + software_version="" +else + software_version_trailing="-${github_branch}-${GITHUB_RUN_ID}" +fi software_version=`python3 ${project_root_dir}/setup.py --version` -echo "software_version=${software_version}" >> ${GITHUB_ENV} +cat ${project_root_dir}/setup.py +echo "software_version=${software_version}${software_version_trailing}" >> ${GITHUB_ENV} cat ${GITHUB_ENV} diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py new file mode 100644 index 00000000..6c2227ff --- /dev/null +++ b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py @@ -0,0 +1,57 @@ +import json +import os + +import pystac + +from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer +from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery +from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator + +LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) + + +class CumulusCreateCollectionDapa: + def __init__(self, event): + self.__event = event + self.__request_body = None + self.__cumulus_collection_query = CollectionsQuery('', '') + self.__cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX') + + def start(self): + if 'body' not in self.__event: + raise ValueError(f'missing body in {self.__event}') + self.__request_body = json.loads(self.__event['body']) + LOGGER.debug(f'request body: {self.__request_body}') + validation_result = pystac.Collection.from_dict(self.__request_body).validate() + if not isinstance(validation_result, list): + LOGGER.error(f'request body is not valid STAC collection: {validation_result}') + return { + 'statusCode': 500, + 'body': {'message': f'request body is not valid STAC Collection schema. check details', + 'details': validation_result} + } + try: + cumulus_collection_doc = CollectionTransformer().from_stac(self.__request_body) + creation_result = self.__cumulus_collection_query.create_collection(cumulus_collection_doc, self.__cumulus_lambda_prefix) + if 'status' not in creation_result: + return { + 'statusCode': 500, + 'body': { + 'message': {creation_result} + } + } + except Exception as e: + LOGGER.exception('error while creating new collection in Cumulus') + return { + 'statusCode': 500, + 'body': { + 'message': f'error while creating new collection in Cumulus. check details', + 'details': str(e) + } + } + return { + 'statusCode': 200, + 'body': { + 'message': creation_result + } + } diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py b/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py index 459d4c7d..598fb32b 100644 --- a/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py +++ b/cumulus_lambda_functions/cumulus_collections_dapa/lambda_function.py @@ -1,4 +1,5 @@ from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_collections_dapa import CumulusCollectionsDapa +from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_create_collection_dapa import CumulusCreateCollectionDapa from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator @@ -12,3 +13,8 @@ def lambda_handler(event, context): LambdaLoggerGenerator.remove_default_handlers() # TODO implement return CumulusCollectionsDapa(event).start() + + +def lambda_handler_ingestion(event, context): + LambdaLoggerGenerator.remove_default_handlers() + return CumulusCreateCollectionDapa(event).start() diff --git a/cumulus_lambda_functions/cumulus_stac/collection_transformer.py b/cumulus_lambda_functions/cumulus_stac/collection_transformer.py index fd608660..1232ec5c 100644 --- a/cumulus_lambda_functions/cumulus_stac/collection_transformer.py +++ b/cumulus_lambda_functions/cumulus_stac/collection_transformer.py @@ -1,9 +1,12 @@ import json from datetime import datetime -from urllib.parse import quote_plus +from urllib.parse import quote_plus, urlparse, unquote_plus +import pystac +from pystac import Link from cumulus_lambda_functions.cumulus_stac.stac_transformer_abstract import StacTransformerAbstract +from cumulus_lambda_functions.lib.time_utils import TimeUtils STAC_COLLECTION_SCHEMA = '''{ "$schema": "http://json-schema.org/draft-07/schema#", @@ -281,11 +284,21 @@ class CollectionTransformer(StacTransformerAbstract): - def __init__(self): + def __init__(self, report_to_ems:bool = True, include_date_range=False): self.__stac_collection_schema = json.loads(STAC_COLLECTION_SCHEMA) self.__cumulus_collection_schema = {} + self.__report_to_ems = report_to_ems + self.__include_date_range = include_date_range - def __convert_to_stac_links(self, collection_file_obj: dict): + def generate_target_link_url(self, regex: str = None, bucket: str = None): + href_link = ['unknown_bucket', 'unknown_regex'] + if regex is not None and regex != '': + href_link[1] = regex + if bucket is not None and bucket != '': + href_link[0] = bucket + return f"./collection.json?bucket={href_link[0]}®ex={quote_plus(href_link[1])}" + + def __convert_to_stac_links(self, collection_file_obj: dict, rel_type: str = 'item'): """ expected output { @@ -310,18 +323,16 @@ def __convert_to_stac_links(self, collection_file_obj: dict): if collection_file_obj is None: return {} stac_link = { - 'rel': 'item', + 'rel': rel_type, } if 'type' in collection_file_obj: stac_link['type'] = collection_file_obj['type'] if 'sampleFileName' in collection_file_obj: stac_link['title'] = collection_file_obj['sampleFileName'] - href_link = ['unknown_bucket', 'unknown_regex'] - if 'bucket' in collection_file_obj: - href_link[0] = collection_file_obj['bucket'] - if 'regex' in collection_file_obj: - href_link[1] = collection_file_obj['regex'] - stac_link['href'] = f"./collection.json?bucket={href_link[0]}®ex={quote_plus(href_link[1])}" + stac_link['href'] = self.generate_target_link_url( + collection_file_obj['regex'] if 'regex' in collection_file_obj else None, + collection_file_obj['bucket'] if 'bucket' in collection_file_obj else None, + ) return stac_link # def to_pystac_link_obj(self, input_dict: dict): @@ -418,14 +429,77 @@ def to_stac(self, source: dict) -> dict: "process": [source['process'] if 'process' in source else ''], "totalGranules": [source['total_size'] if 'total_size' in source else -1], }, - "links": [{ - "rel": "root", - "type": "application/json", - "title": f"{source['name']}___{source['version']}", - "href": "./collection.json" - }] + [self.__convert_to_stac_links(k) for k in source['files']], + "links": [self.__convert_to_stac_links({ + "regex": source['url_path'], + "sampleFileName": source['sampleFileName'], + "type": "application/json", + + }, 'root')] + [self.__convert_to_stac_links(k) for k in source['files']], } return stac_collection + def get_href(self, input_href: str): + parse_result = urlparse(input_href) + if parse_result.query == '': + return '' + query_dict = [k.split('=') for k in parse_result.query.split('&')] + query_dict = {k[0]: unquote_plus(k[1]) for k in query_dict} + return query_dict + + def __convert_from_stac_links(self, link_obj: dict): + output_file_object = { + 'reportToEms': self.__report_to_ems + } + if 'type' in link_obj: + output_file_object['type'] = link_obj['type'] + if 'title' in link_obj: + output_file_object['sampleFileName'] = link_obj['title'] + if 'href' in link_obj: + href_dict = self.get_href(link_obj['href']) + if 'bucket' in href_dict: + output_file_object['bucket'] = href_dict['bucket'] + if 'regex' in href_dict: + output_file_object['regex'] = href_dict['regex'] + return output_file_object + def from_stac(self, source: dict) -> dict: - return {} + input_dapa_collection = pystac.Collection.from_dict(source) + if not input_dapa_collection.validate(): + raise ValueError(f'invalid source dapa: {input_dapa_collection}') + output_collection_cumulus = { + # "createdAt": 1647992847582, + "reportToEms": self.__report_to_ems, + "duplicateHandling": "skip", + # "updatedAt": 1647992847582, + # "timestamp": 1647992849273 + } + summaries = input_dapa_collection.summaries.lists + if 'granuleId' in summaries: + output_collection_cumulus['granuleId'] = summaries['granuleId'][0] + if 'granuleIdExtraction' in summaries: + output_collection_cumulus['granuleIdExtraction'] = summaries['granuleIdExtraction'][0] + if 'process' in summaries: + output_collection_cumulus['process'] = summaries['process'][0] + name_version = input_dapa_collection.id.split('___') + output_collection_cumulus['name'] = name_version[0] + output_collection_cumulus['version'] = name_version[1] + output_files = [] + for each_link_obj in input_dapa_collection.links: + each_link_obj: Link = each_link_obj + each_file_obj = self.__convert_from_stac_links(each_link_obj.to_dict()) + if each_link_obj.rel == 'root': + if 'regex' in each_file_obj: + output_collection_cumulus['url_path'] = each_file_obj['regex'] + if 'sampleFileName' in each_file_obj: + output_collection_cumulus['sampleFileName'] = each_file_obj['sampleFileName'] + else: + output_files.append(each_file_obj) + output_collection_cumulus['files'] = output_files + if len(input_dapa_collection.extent.temporal.intervals) > 0: + date_interval = input_dapa_collection.extent.temporal.intervals[0] + if len(date_interval) == 2 and self.__include_date_range is True: + if date_interval[0] is not None: + output_collection_cumulus['dateFrom'] = date_interval[0].strftime(TimeUtils.MMDD_FORMAT) + if date_interval[1] is not None: + output_collection_cumulus['dateTo'] = date_interval[1].strftime(TimeUtils.MMDD_FORMAT) + return output_collection_cumulus diff --git a/cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py b/cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py new file mode 100644 index 00000000..df1245bd --- /dev/null +++ b/cumulus_lambda_functions/cumulus_stac/unity_collection_stac.py @@ -0,0 +1,69 @@ +from datetime import datetime + +from pystac import Link, Collection, Extent, SpatialExtent, TemporalExtent, Summaries + +from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer +from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator + +LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) + + +class UnityCollectionStac: + def __init__(self): + self.__id = '' + self.__granule_id_extraction_regex = '' + self.__process = '' + self.__collection_title = '' + self.__granule_id_regex = '' + self.__sample_filename = '' + self.__files = [] + self.__collection_transformer = CollectionTransformer() + + def with_title(self, title: str): + self.__collection_title = title + return self + + def with_process(self, process: str): + self.__process = process + return self + + def with_id(self, collection_id: str): + self.__id = collection_id + if '___' not in collection_id: + LOGGER.warning(f'no ID in {collection_id}. using 001') + self.__id = f'{self.__id}___001' + return self + + def with_graule_id_regex(self, granule_id_regex): + self.__granule_id_regex = granule_id_regex + return self + + def with_granule_id_extraction_regex(self, granule_id_extraction_regex): + self.__granule_id_extraction_regex = granule_id_extraction_regex + return self + + def add_file_type(self, title: str, regex: str, bucket: str, media_type: str, rel: str = 'item'): + if rel == 'root': + LOGGER.debug('updating media_type for rel = root') + media_type = 'application/json' + self.__files.append(Link(rel=rel, target=self.__collection_transformer.generate_target_link_url(regex, bucket), media_type=media_type, title=title)) + return self + + def start(self): + # TODO validate + stac_collection = Collection(id=self.__id, + description='TODO', + extent=Extent(SpatialExtent([[0, 0, 0, 0]]), + TemporalExtent([[datetime.utcnow(), datetime.utcnow()]])), + title=self.__collection_title, + summaries=Summaries({ + 'granuleId': [self.__granule_id_regex], + 'granuleIdExtraction': [self.__granule_id_extraction_regex], + 'process': [self.__process] + }), + ) + stac_collection.add_links(self.__files) + new_collection = stac_collection.to_dict(include_self_link=False) + if 'links' in new_collection and len(new_collection['links']) > 0 and new_collection['links'][0]['rel'] == 'root': + new_collection['links'][0]['href'] = './collection.json' + return new_collection diff --git a/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py b/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py index 1c5a79f1..7d22c649 100644 --- a/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py +++ b/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse, unquote_plus from cumulus_lambda_functions.cumulus_dapa_client.dapa_client import DapaClient +from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3 LOGGER = logging.getLogger(__name__) @@ -49,11 +50,7 @@ def __set_props_from_env(self): return self def __get_href(self, input_href: str): - parse_result = urlparse(input_href) - if parse_result.query == '': - return '' - query_dict = [k.split('=') for k in parse_result.query.split('&')] - query_dict = {k[0]: unquote_plus(k[1]) for k in query_dict} + query_dict = CollectionTransformer().get_href(input_href) if 'regex' not in query_dict: raise ValueError(f'missing regex in {input_href}') return query_dict['regex'] diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_collections.py b/cumulus_lambda_functions/cumulus_wrapper/query_collections.py index 8788b52c..b19eaf9e 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_collections.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_collections.py @@ -46,6 +46,36 @@ def get_size(self, private_api_prefix: str): total_size = query_result['meta']['count'] return {'total_size': total_size} + def create_collection(self, new_collection: dict, private_api_prefix: str): + payload = { + 'httpMethod': 'POST', + 'resource': '/{proxy+}', + 'path': f'/{self.__collections_key}', + 'headers': { + 'Content-Type': 'application/json', + }, + 'body': json.dumps(new_collection) + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 500, 'body': '', 'headers': {}} + """ + if query_result['statusCode'] >= 500: + LOGGER.error(f'server error status code: {query_result["statusCode"]}. details: {query_result}') + return {'server_error': query_result} + if query_result['statusCode'] >= 400: + LOGGER.error(f'client error status code: {query_result["statusCode"]}. details: {query_result}') + return {'client_error': query_result} + query_result = json.loads(query_result['body']) + LOGGER.debug(f'json query_result: {query_result}') + if 'message' not in query_result: + return {'server_error': f'invalid response: {query_result}'} + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + return {'status': query_result['message']} def __get_stats(self, collection_id, private_api_prefix: str): payload = { diff --git a/setup.py b/setup.py index 0c63f6f9..bef7c4d6 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ ] flask_requires = [ + 'pystac', 'jsonschema', 'flask===2.0.1', 'flask_restful===0.3.9', 'flask-restx===0.5.0', # to create Flask server 'gevent===21.8.0', 'greenlet===1.1.1', # to run flask server 'werkzeug===2.0.1', diff --git a/tests/cumulus_lambda_functions/cumulus_collections_dapa/__init__.py b/tests/cumulus_lambda_functions/cumulus_collections_dapa/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py b/tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py new file mode 100644 index 00000000..f0e0b838 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_collections_dapa/test_cumulus_create_collection_dapa.py @@ -0,0 +1,32 @@ +import json +import os +from datetime import datetime +from unittest import TestCase + +from cumulus_lambda_functions.cumulus_collections_dapa.cumulus_create_collection_dapa import CumulusCreateCollectionDapa +from cumulus_lambda_functions.cumulus_stac.unity_collection_stac import UnityCollectionStac + + +class TestCumulusCreateCollectionDapa(TestCase): + def test_01(self): + dapa_collection = UnityCollectionStac() \ + .with_id(f'CUMULUS_DAPA_UNIT_TEST___{int(datetime.utcnow().timestamp())}') \ + .with_graule_id_regex("^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$") \ + .with_granule_id_extraction_regex("(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+") \ + .with_title("P1570515ATMSSCIENCEAXT11344000000001.PDS") \ + .with_process('modis') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + 'internal', 'data', 'item') + os.environ['CUMULUS_LAMBDA_PREFIX'] = 'am-uds-dev-cumulus' + stac_collection = dapa_collection.start() + event = { + 'body': json.dumps(stac_collection) + } + creation = CumulusCreateCollectionDapa(event).start() + self.assertTrue('statusCode' in creation, f'missing statusCode: {creation}') + self.assertEqual(200, creation['statusCode'], f'wrong statusCode: {creation}') + return diff --git a/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py b/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py index 9968ea66..af4a67e0 100644 --- a/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py +++ b/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py @@ -51,12 +51,12 @@ def test_01(self): "granuleIdExtraction": "(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+", "reportToEms": True, "version": "001", - "duplicateHandling": "replace", + "duplicateHandling": "skip", "updatedAt": 1647992847582, "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", "timestamp": 1647992849273 } - raw = { + converted_stac = { "type": "Collection", "stac_version": "1.0.0", # "stac_extensions": [], @@ -82,6 +82,14 @@ def test_01(self): }, ] } - raw = CollectionTransformer().to_stac(source) - self.assertEqual(None, stac_validator.validate(raw), f'invalid stac format: {stac_validator}') + converted_stac = CollectionTransformer(include_date_range=True).to_stac(source) + self.assertEqual(None, stac_validator.validate(converted_stac), f'invalid stac format: {stac_validator}') + converted_cumulus = CollectionTransformer(include_date_range=True).from_stac(converted_stac) + for k, v in source.items(): + if k in ['updatedAt', 'timestamp', 'createdAt']: + continue + self.assertTrue(k in converted_cumulus, f'missing {k}') + if k not in ['files', 'dateFrom', 'dateTo']: + self.assertEqual(v, converted_cumulus[k], f'wrong value for {k}') + self.assertEqual(sorted(json.dumps(source['files'])), sorted(json.dumps(converted_cumulus['files'])), f"wrong files content: {source['files']} vs. {converted_cumulus['files']}") return diff --git a/tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py b/tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py new file mode 100644 index 00000000..df970581 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_stac/test_unity_collection_stac.py @@ -0,0 +1,26 @@ +import json +from datetime import datetime +from unittest import TestCase + +import pystac + +from cumulus_lambda_functions.cumulus_stac.unity_collection_stac import \ + UnityCollectionStac + + +class TestUnityCollectionStac(TestCase): + def test_01(self): + dapa_collection = UnityCollectionStac()\ + .with_id(f'CUMULUS_DAPA_UNIT_TEST___{int(datetime.utcnow().timestamp())}')\ + .with_graule_id_regex("^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$")\ + .with_granule_id_extraction_regex("(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+")\ + .with_title("P1570515ATMSSCIENCEAXT11344000000001.PDS")\ + .with_process('modis')\ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", 'internal', 'metadata', 'item') \ + .add_file_type("P1570515ATMSSCIENCEAXT11344000000000.PDS", "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", 'internal', 'data', 'item') + + stac_collection = dapa_collection.start() + validation_result = pystac.Collection.from_dict(stac_collection).validate() + self.assertTrue(isinstance(validation_result, list), f'wrong validation for : {json.dumps(stac_collection, indent=4)}. details: {validation_result}') + return diff --git a/tests/cumulus_lambda_functions/cumulus_wrapper/__init__.py b/tests/cumulus_lambda_functions/cumulus_wrapper/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py b/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py new file mode 100644 index 00000000..7d2de4d0 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_wrapper/test_query_collection.py @@ -0,0 +1,71 @@ +from datetime import datetime +from unittest import TestCase + +from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery + + +class TestQueryCollection(TestCase): + def test_01(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + collection_version = int(datetime.utcnow().timestamp()) + sample_collection = { + # "dataType": "MOD09GQ", + # "provider_path": "cumulus-test-data/pdrs", + "name": "UNITY_CUMULUS_DEV_UNIT_TEST", + "version": str(collection_version), + # "process": "modis", + # "duplicateHandling": "skip", + "granuleId": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$", + "granuleIdExtraction": "(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+", + # "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "files": [ + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS", + "type": "data", + "reportToEms": True + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "reportToEms": True, + "type": "metadata" + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "reportToEms": True, + "type": "metadata" + }, + { + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "reportToEms": True, + "type": "metadata" + } + ], + } + # sample_collection = { + # "createdAt": 1647992847582, + # "reportToEms": True, + # "updatedAt": 1647992847582, + # "timestamp": 1647992849273 + # } + response = collection_query.create_collection(sample_collection, lambda_prefix) + self.assertTrue('status' in response, f'status not in response: {response}') + self.assertEqual('Record saved', response['status'], f'wrong status: {response}') + return + + def test_02(self): + lambda_prefix = 'am-uds-dev-cumulus' + collection_query = CollectionsQuery('NA', 'NA') + + collections = collection_query.query_direct_to_private_api(lambda_prefix) + self.assertTrue('results' in collections, f'results not in collections: {collections}') + return diff --git a/tf-module/unity-cumulus/main.tf b/tf-module/unity-cumulus/main.tf index 33fd759d..d6cf6d60 100644 --- a/tf-module/unity-cumulus/main.tf +++ b/tf-module/unity-cumulus/main.tf @@ -38,7 +38,7 @@ resource "aws_lambda_function" "snpp_lvl0_generate_cmr" { function_name = "${var.prefix}-snpp_lvl0_generate_cmr" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.snpp_lvl0_generate_cmr.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { variables = { @@ -58,7 +58,7 @@ resource "aws_lambda_function" "snpp_lvl1_generate_cmr" { function_name = "${var.prefix}-snpp_lvl1_generate_cmr" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.snpp_level1a_generate_cmr.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { variables = { @@ -78,7 +78,7 @@ resource "aws_lambda_function" "cumulus_granules_dapa" { function_name = "${var.prefix}-cumulus_granules_dapa" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.cumulus_granules_dapa.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { @@ -100,7 +100,7 @@ resource "aws_lambda_function" "cumulus_collections_dapa" { function_name = "${var.prefix}-cumulus_collections_dapa" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.cumulus_collections_dapa.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { @@ -122,7 +122,7 @@ resource "aws_lambda_function" "cumulus_collections_ingest_cnm_dapa" { function_name = "${var.prefix}-cumulus_collections_ingest_cnm_dapa" role = var.lambda_processing_role_arn handler = "cumulus_lambda_functions.cumulus_granules_dapa_ingest_cnm.lambda_function.lambda_handler" - runtime = "python3.7" + runtime = "python3.9" timeout = 300 environment { @@ -132,6 +132,28 @@ resource "aws_lambda_function" "cumulus_collections_ingest_cnm_dapa" { } } + vpc_config { + subnet_ids = var.cumulus_lambda_subnet_ids + security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id] + } + tags = var.tags +} + +resource "aws_lambda_function" "cumulus_collections_creation_dapa" { + filename = local.lambda_file_name + function_name = "${var.prefix}-cumulus_collections_creation_dapa" + role = var.lambda_processing_role_arn + handler = "cumulus_lambda_functions.cumulus_granules_dapa_ingest_cnm.lambda_function.lambda_handler_ingestion" + runtime = "python3.9" + timeout = 300 + + environment { + variables = { + LOG_LEVEL = var.log_level + CUMULUS_LAMBDA_PREFIX = var.prefix + } + } + vpc_config { subnet_ids = var.cumulus_lambda_subnet_ids security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id]