diff --git a/.gitignore b/.gitignore index d85ef2c7..5126e30b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ scratch* local* *egg-info* dist -__pycache__ \ No newline at end of file +__pycache__ +.env \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index afac60fe..f6137596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.7.0] - 2022-09-06 +### Added +- [#62](https://github.com/unity-sds/unity-data-services/issues/66) Added OpenAPI spec for DAPA endpoints +- [#66](https://github.com/unity-sds/unity-data-services/issues/66) Added pagination links to STAC response of DAPA endpoints +- [#64](https://github.com/unity-sds/unity-data-services/issues/64) Added temporal coverage to DAPA collection endpoint +### Changed +- [#67](https://github.com/unity-sds/unity-data-services/issues/67) Updated STAC collection schema to be compatible with PySTAC library +### Fixed + ## [1.6.17] - 2022-07-28 ### Added ### Fixed diff --git a/ci.cd/Makefile b/ci.cd/Makefile index c2f12a28..f9ff7556 100644 --- a/ci.cd/Makefile +++ b/ci.cd/Makefile @@ -4,7 +4,7 @@ export VERSION ?= latest all: build_lambda upload_lambda update_lambda_function build_docker - +local: build_lambda upload_lambda update_lambda_function_1 update_lambda_function_2 update_lambda_function_3 build_docker: docker build -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile . @@ -12,13 +12,16 @@ zip_docker: docker save "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" | gzip > "$(NAME)__$(VERSION).tar.gz" build_lambda: - docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.7 ci.cd/create_s3_zip.sh + docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.9 ci.cd/create_s3_zip.sh build_lambda_public: docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" python:3.7 ci.cd/create_s3_zip.sh upload_lambda: aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/ - -update_lambda_function: - aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:Test1 --publish +update_lambda_function_1: + aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_dapa --publish &>/dev/null +update_lambda_function_2: + aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_granules_dapa --publish &>/dev/null +update_lambda_function_3: + aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa --publish &>/dev/null diff --git a/ci.cd/create_s3_zip.sh b/ci.cd/create_s3_zip.sh index 876cb480..5d293938 100755 --- a/ci.cd/create_s3_zip.sh +++ b/ci.cd/create_s3_zip.sh @@ -8,7 +8,7 @@ zip_file="${project_root_dir}/$ZIP_NAME" ; # save the result file in current wor tmp_proj='/tmp/cumulus_lambda_functions' -source_dir="/usr/local/lib/python3.7/site-packages/" +source_dir="/usr/local/lib/python3.9/site-packages/" mkdir -p "$tmp_proj/cumulus_lambda_functions" && \ cd $tmp_proj && \ diff --git a/cognito_readme.md b/cognito_readme.md index 1c10e235..c889feec 100644 --- a/cognito_readme.md +++ b/cognito_readme.md @@ -9,9 +9,11 @@ "ClientId" : "7a1fglm2d54eoggj13lccivp25" } - ask U-CS to create credentials and change password the first time -- run this command: +- run this command (JPL AWS): curl -X POST --data @cognito.jpl.aws.json -H 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' -H 'Content-Type: application/x-amz-json-1.1' https://cognito-idp.us-west-2.amazonaws.com/|jq + curl -X POST --data @cognito.mcp.test.aws.json -H 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' -H 'Content-Type: application/x-amz-json-1.1' https://cognito-idp.us-west-2.amazonaws.com/|jq + curl -X POST --data @cognito.mcp.dev.aws.json -H 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' -H 'Content-Type: application/x-amz-json-1.1' https://cognito-idp.us-west-2.amazonaws.com/|jq - successful response: { diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py index 2abaab5a..6868ba3e 100644 --- a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py +++ b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py @@ -3,6 +3,7 @@ from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator +from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) @@ -37,25 +38,50 @@ def __assign_values(self): self.__offset = int(query_str_dict['offset']) return self + def __get_size(self): + try: + cumulus_size = self.__cumulus.get_size(self.__cumulus_lambda_prefix) + except: + LOGGER.exception(f'cannot get cumulus_size') + cumulus_size = {'total_size': -1} + return cumulus_size + + def __get_pagination_urls(self): + try: + pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links() + except Exception as e: + LOGGER.exception(f'error while generating pagination links') + return [{'message': f'error while generating pagination links: {str(e)}'}] + return pagination_links + def start(self): try: cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix) - except Exception as e: + if 'server_error' in cumulus_result: + return { + 'statusCode': 500, + 'body': {'message': cumulus_result['server_error']} + } + if 'client_error' in cumulus_result: + return { + 'statusCode': 400, + 'body': {'message': cumulus_result['client_error']} + } + cumulus_size = self.__get_size() return { - 'statusCode': 500, - 'body': {'message': f'unpredicted error: {str(e)}'} + 'statusCode': 200, + 'body': json.dumps({ + 'numberMatched': cumulus_size['total_size'], + 'numberReturned': len(cumulus_result['results']), + 'stac_version': '1.0.0', + 'type': 'FeatureCollection', + 'links': self.__get_pagination_urls(), + 'features': cumulus_result['results'], + }) } - if 'server_error' in cumulus_result: + except Exception as e: + LOGGER.exception(f'unexpected error') return { 'statusCode': 500, - 'body': {'message': cumulus_result['server_error']} - } - if 'client_error' in cumulus_result: - return { - 'statusCode': 400, - 'body': {'message': cumulus_result['client_error']} + 'body': {'message': f'unpredicted error: {str(e)}'} } - return { - 'statusCode': 200, - 'body': json.dumps({'features': cumulus_result['results']}) - } diff --git a/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py b/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py index 7c3bf4d4..124af005 100644 --- a/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py +++ b/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py @@ -3,6 +3,7 @@ from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator +from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) @@ -87,25 +88,50 @@ def __assign_values(self): self.__offset = int(query_str_dict['offset']) return self + def __get_size(self): + try: + cumulus_size = self.__cumulus.get_size(self.__cumulus_lambda_prefix) + except: + LOGGER.exception(f'cannot get cumulus_size') + cumulus_size = {'total_size': -1} + return cumulus_size + + def __get_pagination_urls(self): + try: + pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links() + except Exception as e: + LOGGER.exception(f'error while generating pagination links') + return [{'message': f'error while generating pagination links: {str(e)}'}] + return pagination_links + def start(self): try: cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix) - except Exception as e: + if 'server_error' in cumulus_result: + return { + 'statusCode': 500, + 'body': {'message': cumulus_result['server_error']} + } + if 'client_error' in cumulus_result: + return { + 'statusCode': 400, + 'body': {'message': cumulus_result['client_error']} + } + cumulus_size = self.__get_size() return { - 'statusCode': 500, - 'body': {'message': f'unpredicted error: {str(e)}'} + 'statusCode': 200, + 'body': json.dumps({ + 'numberMatched': cumulus_size['total_size'], + 'numberReturned': len(cumulus_result['results']), + 'stac_version': '1.0.0', + 'type': 'FeatureCollection', # TODO correct name? + 'links': self.__get_pagination_urls(), + 'features': cumulus_result['results'] + }) } - if 'server_error' in cumulus_result: + except Exception as e: + LOGGER.exception(f'unexpected error') return { 'statusCode': 500, - 'body': {'message': cumulus_result['server_error']} - } - if 'client_error' in cumulus_result: - return { - 'statusCode': 400, - 'body': {'message': cumulus_result['client_error']} + 'body': {'message': f'unpredicted error: {str(e)}'} } - return { - 'statusCode': 200, - 'body': json.dumps({'features': cumulus_result['results']}) - } diff --git a/cumulus_lambda_functions/cumulus_stac/collection_transformer.py b/cumulus_lambda_functions/cumulus_stac/collection_transformer.py index b50971e4..fd608660 100644 --- a/cumulus_lambda_functions/cumulus_stac/collection_transformer.py +++ b/cumulus_lambda_functions/cumulus_stac/collection_transformer.py @@ -1,4 +1,7 @@ import json +from datetime import datetime +from urllib.parse import quote_plus + from cumulus_lambda_functions.cumulus_stac.stac_transformer_abstract import StacTransformerAbstract @@ -318,9 +321,12 @@ def __convert_to_stac_links(self, collection_file_obj: dict): href_link[0] = collection_file_obj['bucket'] if 'regex' in collection_file_obj: href_link[1] = collection_file_obj['regex'] - stac_link['href'] = '___'.join(href_link) + stac_link['href'] = f"./collection.json?bucket={href_link[0]}®ex={quote_plus(href_link[1])}" return stac_link + # def to_pystac_link_obj(self, input_dict: dict): + # return + def to_stac(self, source: dict) -> dict: source_sample = { "createdAt": 1647992847582, @@ -366,6 +372,26 @@ def to_stac(self, source: dict) -> dict: "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", "timestamp": 1647992849273 } + # TemporalIntervals([ + # datetime.strptime(source['dateFrom']) + # ]) + # stac_collection = pystac.Collection( + # id=f"{source['name']}___{source['version']}", + # description='TODO', + # extent=Extent( + # SpatialExtent([[0, 0, 0, 0]]), + # TemporalExtent([[source['dateFrom'] if 'dateFrom' in source else None, + # source['dateTo'] if 'dateTo' in source else None]]) + # ), + # summaries=Summaries({ + # "granuleId": [source['granuleId'] if 'granuleId' in source else ''], + # "granuleIdExtraction": [source['granuleIdExtraction'] if 'granuleIdExtraction' in source else ''], + # "process": [source['process'] if 'process' in source else ''], + # "totalGranules": [source['total_size'] if 'total_size' in source else -1], + # }), + # ) + # stac_collection.get_root_link().target = './collection.json' + # stac_collection.add_links([Link.from_dict(k) for k in [self.__convert_to_stac_links(k) for k in source['files']]]) stac_collection = { "type": "Collection", "stac_version": "1.0.0", @@ -380,16 +406,24 @@ def to_stac(self, source: dict) -> dict: "bbox": [[0, 0, 0, 0]] }, "temporal": { - "interval": [[None, None]] + "interval": [[source['dateFrom'] if 'dateFrom' in source else None, + source['dateTo'] if 'dateTo' in source else None + ]] } }, "assets": {}, "summaries": { - "granuleId": source['granuleId'] if 'granuleId' in source else '', - "granuleIdExtraction": source['granuleIdExtraction'] if 'granuleIdExtraction' in source else '', - "process": source['process'] if 'process' in source else '', + "granuleId": [source['granuleId'] if 'granuleId' in source else ''], + "granuleIdExtraction": [source['granuleIdExtraction'] if 'granuleIdExtraction' in source else ''], + "process": [source['process'] if 'process' in source else ''], + "totalGranules": [source['total_size'] if 'total_size' in source else -1], }, - "links": [self.__convert_to_stac_links(k) for k in source['files']], + "links": [{ + "rel": "root", + "type": "application/json", + "title": f"{source['name']}___{source['version']}", + "href": "./collection.json" + }] + [self.__convert_to_stac_links(k) for k in source['files']], } return stac_collection diff --git a/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py b/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py index 5489758c..1c5a79f1 100644 --- a/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py +++ b/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py @@ -1,11 +1,9 @@ -import json import logging import os import re from collections import defaultdict from glob import glob - -import requests +from urllib.parse import urlparse, unquote_plus from cumulus_lambda_functions.cumulus_dapa_client.dapa_client import DapaClient from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3 @@ -50,9 +48,19 @@ def __set_props_from_env(self): self.__delete_files = os.environ.get(self.DELETE_FILES_KEY, 'FALSE').strip().upper() == 'TRUE' return self + def __get_href(self, input_href: str): + parse_result = urlparse(input_href) + if parse_result.query == '': + return '' + query_dict = [k.split('=') for k in parse_result.query.split('&')] + query_dict = {k[0]: unquote_plus(k[1]) for k in query_dict} + if 'regex' not in query_dict: + raise ValueError(f'missing regex in {input_href}') + return query_dict['regex'] + def __sort_granules(self): - file_regex_list = {k['type']: k['href'].split('___')[-1] for k in self.__collection_details['links'] if not k['title'].endswith('cmr.xml')} - granule_id_extraction = self.__collection_details['summaries']['granuleIdExtraction'] + file_regex_list = {k['type']: self.__get_href(k['href']) for k in self.__collection_details['links'] if k['rel'] != 'root' and not k['title'].endswith('cmr.xml')} + granule_id_extraction = self.__collection_details['summaries']['granuleIdExtraction'][0] granules = defaultdict(dict) for each_file in self.__raw_files: each_filename = os.path.basename(each_file) diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_collections.py b/cumulus_lambda_functions/cumulus_wrapper/query_collections.py index dad53e80..8788b52c 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_collections.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_collections.py @@ -11,10 +11,88 @@ class CollectionsQuery(CumulusBase): __collections_key = 'collections' + __stats_key = 'stats' def __init__(self, cumulus_base: str, cumulus_token: str): super().__init__(cumulus_base, cumulus_token) + def get_size(self, private_api_prefix: str): + payload = { + 'httpMethod': 'GET', + 'resource': '/{proxy+}', + 'path': f'/stats/aggregate', + 'queryStringParameters': {'field': 'status', 'type': 'collections'}, + 'headers': { + 'Content-Type': 'application/json', + }, + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False} + """ + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + if query_result['statusCode'] >= 500: + raise ValueError(f'server_error: {query_result.statusCode}. details: {query_result}') + if query_result['statusCode'] >= 400: + raise ValueError(f'client_error: {query_result.statusCode}. details: {query_result}') + query_result = json.loads(query_result['body']) + LOGGER.info(f'json query_result: {query_result}') + if 'meta' not in query_result or 'count' not in query_result['meta']: + raise ValueError(f'server_error: missing key: results. invalid response json: {query_result}') + total_size = query_result['meta']['count'] + return {'total_size': total_size} + + + def __get_stats(self, collection_id, private_api_prefix: str): + payload = { + 'httpMethod': 'GET', + 'resource': '/{proxy+}', + 'path': f'/{self.__collections_key}', + 'queryStringParameters': {k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]}, + } + try: + query_stats_result = self._invoke_api(payload, private_api_prefix) + except: + LOGGER.exception(f'error while trying to retrieve stats for collection: {collection_id}') + return {} + if query_stats_result['statusCode'] >= 400: + return {} + return query_stats_result['results'] + + def get_stats(self, collection_id:str, private_api_prefix: str): + payload = { + 'httpMethod': 'GET', + 'resource': '/{proxy+}', + 'path': f'/stats', + 'queryStringParameters': {'type': 'granules', 'collectionId': collection_id}, + 'headers': { + 'Content-Type': 'application/json', + }, + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False} + """ + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + if query_result['statusCode'] >= 500: + raise ValueError(f'server_error: {query_result.statusCode}. details: {query_result}') + if query_result['statusCode'] >= 400: + raise ValueError(f'client_error: {query_result.statusCode}. details: {query_result}') + query_result = json.loads(query_result['body']) + LOGGER.info(f'json query_result: {query_result}') + if 'granules' not in query_result: + raise ValueError(f'server_error: missing key: results. invalid response json: {query_result}') + stats = query_result['granules'] + return stats + def query_direct_to_private_api(self, private_api_prefix: str): payload = { 'httpMethod': 'GET', @@ -40,6 +118,12 @@ def query_direct_to_private_api(self, private_api_prefix: str): LOGGER.error(f'missing key: results. invalid response json: {query_result}') return {'server_error': f'missing key: results. invalid response json: {query_result}'} query_result = query_result['results'] + for each_collection in query_result: + stac_collection_id = f"{each_collection['name']}___{each_collection['version']}" + stats = self.get_stats(stac_collection_id, private_api_prefix) + each_collection['dateFrom'] = stats['dateFrom'] + each_collection['dateTo'] = stats['dateTo'] + each_collection['total_size'] = stats['value'] stac_list = [CollectionTransformer().to_stac(k) for k in query_result] LOGGER.debug(f'stac_list: {stac_list}') except Exception as e: diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py index 7fdfeccd..caffc365 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py @@ -51,6 +51,36 @@ def with_time_range(self, from_time, to_time): self._conditions.append(f'{self.__beginning_time_key}__to={to_time}') return self + def get_size(self, private_api_prefix: str): + payload = { + 'httpMethod': 'GET', + 'resource': '/{proxy+}', + 'path': f'/stats/aggregate', + 'queryStringParameters': {**{k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]}, **{'field': 'status', 'type': 'granules'}}, + 'headers': { + 'Content-Type': 'application/json', + }, + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False} + """ + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + if query_result['statusCode'] >= 500: + raise ValueError(f'server_error: {query_result.statusCode}. details: {query_result}') + if query_result['statusCode'] >= 400: + raise ValueError(f'client_error: {query_result.statusCode}. details: {query_result}') + query_result = json.loads(query_result['body']) + LOGGER.info(f'json query_result: {query_result}') + if 'meta' not in query_result or 'count' not in query_result['meta']: + raise ValueError(f'server_error: missing key: results. invalid response json: {query_result}') + total_size = query_result['meta']['count'] + return {'total_size': total_size} + def query_direct_to_private_api(self, private_api_prefix: str): payload = { 'httpMethod': 'GET', diff --git a/cumulus_lambda_functions/lib/cognito_login/cognito_login.py b/cumulus_lambda_functions/lib/cognito_login/cognito_login.py index 5dda5682..074e0232 100644 --- a/cumulus_lambda_functions/lib/cognito_login/cognito_login.py +++ b/cumulus_lambda_functions/lib/cognito_login/cognito_login.py @@ -130,6 +130,6 @@ def start(self, username: str, dwssap: str): login_result = json.loads(response.content.decode('utf-8')) result = JsonValidator(COGNITO_RESULT_SCHEMA).validate(login_result) if result is not None: - raise ValueError(f'pds metadata has validation errors: {result}') + raise ValueError(f'cognito login response has validation errors: {result}') self.__token = login_result['AuthenticationResult']['AccessToken'] return self diff --git a/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py b/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py new file mode 100644 index 00000000..c7d216e8 --- /dev/null +++ b/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py @@ -0,0 +1,107 @@ +from copy import deepcopy + +from cumulus_lambda_functions.lib.json_validator import JsonValidator +from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator + +API_GATEWAY_EVENT_SCHEMA = { + 'type': 'object', + 'required': [ + 'requestContext', + 'headers', + ], + 'properties': { + 'requestContext': { + 'type': 'object', + 'required': [ + 'path', + ], + 'properties': { + 'path': { + 'type': 'string' + } + } + }, + 'headers': { + 'type': 'object', + 'required': [ + 'Host', + ], + 'properties': { + 'Host': { + 'type': 'string' + } + } + } + } +} +LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) + + +class LambdaApiGatewayUtils: + def __init__(self, event: dict, default_limit: int = 10): + self.__event = event + self.__default_limit = default_limit + api_gateway_event_validator_result = JsonValidator(API_GATEWAY_EVENT_SCHEMA).validate(event) + if api_gateway_event_validator_result is not None: + raise ValueError(f'invalid event: {api_gateway_event_validator_result}. event: {event}') + + def __get_current_page(self): + try: + requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}" + new_queries = deepcopy(self.__event['queryStringParameters']) if 'queryStringParameters' in self.__event and self.__event[ + 'queryStringParameters'] is not None else {} + limit = int(new_queries['limit'] if 'limit' in new_queries else self.__default_limit) + offset = int(new_queries['offset'] if 'offset' in new_queries else 0) + new_queries['limit'] = limit + new_queries['offset'] = offset + requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}" + except Exception as e: + LOGGER.exception(f'error while getting current page URL') + return f'unable to get current page URL, {str(e)}' + return requesting_url + + def __get_next_page(self): + try: + requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}" + new_queries = deepcopy(self.__event['queryStringParameters']) if 'queryStringParameters' in self.__event and self.__event[ + 'queryStringParameters'] is not None else {} + limit = int(new_queries['limit'] if 'limit' in new_queries else self.__default_limit) + if limit == 0: + return '' + offset = int(new_queries['offset'] if 'offset' in new_queries else 0) + offset += limit + new_queries['limit'] = limit + new_queries['offset'] = offset + requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}" + except Exception as e: + LOGGER.exception(f'error while getting next page URL') + return f'unable to get next page URL, {str(e)}' + return requesting_url + + def __get_prev_page(self): + try: + requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}" + new_queries = deepcopy(self.__event['queryStringParameters']) if 'queryStringParameters' in self.__event and self.__event[ + 'queryStringParameters'] is not None else {} + limit = int(new_queries['limit'] if 'limit' in new_queries else self.__default_limit) + if limit == 0: + return '' + offset = int(new_queries['offset'] if 'offset' in new_queries else 0) + offset -= limit + if offset < 0: + offset = 0 + new_queries['limit'] = limit + new_queries['offset'] = offset + requesting_url = f"{requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}" + except Exception as e: + LOGGER.exception(f'error while getting previous page URL') + return f'unable to get previous page URL, {str(e)}' + return requesting_url + + def generate_pagination_links(self): + return [ + {'rel': 'self', 'href': self.__get_current_page()}, + {'rel': 'root', 'href': f"https://{self.__event['headers']['Host']}"}, + {'rel': 'next', 'href': self.__get_next_page()}, + {'rel': 'prev', 'href': self.__get_prev_page()}, + ] diff --git a/docker/docker-compose-granules-download.yaml b/docker/docker-compose-granules-download.yaml index 2293a7f8..4e811da9 100644 --- a/docker/docker-compose-granules-download.yaml +++ b/docker/docker-compose-granules-download.yaml @@ -1,7 +1,7 @@ version: "3.7" services: cumulus_granules_download: - image: ghcr.io/unity-sds/unity-data-services:1.5.15 + image: ghcr.io/unity-sds/unity-data-services:1.7.0 volumes: - /tmp/cumulus_granules:/etc/granules command: ["download"] diff --git a/docker/docker-compose-granules-upload.yaml b/docker/docker-compose-granules-upload.yaml index 6956d741..a830be88 100644 --- a/docker/docker-compose-granules-upload.yaml +++ b/docker/docker-compose-granules-upload.yaml @@ -1,7 +1,7 @@ version: "3.7" services: cumulus_granules_upload: - image: ghcr.io/unity-sds/unity-data-services:1.5.15 + image: ghcr.io/unity-sds/unity-data-services:1.7.0 volumes: - /tmp/snpp_upload_test_1:/etc/snpp_upload_test_1 command: ["upload"] diff --git a/etc/dapa_open_api.yaml b/etc/dapa_open_api.yaml new file mode 100644 index 00000000..97e8b20d --- /dev/null +++ b/etc/dapa_open_api.yaml @@ -0,0 +1,148 @@ +openapi: 3.0.3 +info: + title: in-situ-data-services + description: API for querying in-situ data sources + version: 0.0.1 + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + +servers: + - url: https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev + description: 'JPL AWS Cloud API Gateway' + - url: https://1gp9st60gd.execute-api.us-west-2.amazonaws.com/dev + description: 'MCP AWS Cloud API Gateway - DEV Venue' + - url: https://58nbcawrvb.execute-api.us-west-2.amazonaws.com/test + description: 'MCP AWS Cloud API Gateway - TEST Venue' + +paths: + '/am-uds-dapa/collections': + get: + summary: 'list all collections in Cumulus' + parameters: + - name: "Authorization" + description: "Authorization Bearer token. It should begin with the key `Bearer `" + in: "header" + required: true + schema: + type: "string" + - name: "limit" + description: "How many items in 1 page. defaulted to 10" + in: "query" + required: false + schema: + type: "string" + - name: "offset" + description: "How many items to skip. if limit == 10, offset = 10 for 2nd page, so on and so forth" + in: "query" + required: false + schema: + type: "string" + responses: + '200': + description: 'Successful query' + content: + application/json: + schema: + type: object + properties: + features: + example: 'STAC collection array' + type: array + items: + type: object + '500': + description: 'Server Error' + '403': + description: 'Authorization Error' + '401': + description: 'Authorization Error' + put: + summary: 'ingest new granules to a collection' + parameters: + - name: "Authorization" + description: "Authorization Bearer token. It should begin with the key `Bearer `" + in: "header" + required: true + schema: + type: "string" + requestBody: + description: 'Ingestion Body in JSON' + required: true + content: + application/json: + schema: + type: object + properties: + provider_id: + type: string + description: 'Provider ID defined in Cumulus' + features: + description: 'STAC granules array' + type: array + items: + type: object + responses: + '200': + description: 'Successful query' + content: + application/json: + schema: + type: object + properties: + message: + description: 'result description' + type: string + '500': + description: 'Server Error' + '403': + description: 'Authorization Error' + '401': + description: 'Authorization Error' + '/am-uds-dapa/collections/{collectionId}/items': + get: + summary: 'list all granules in Cumulus' + parameters: + - name: "Authorization" + description: "Authorization Bearer token. It should begin with the key `Bearer `" + in: "header" + required: true + schema: + type: "string" + - name: "collectionId" + description: "Cumulus Collection Id from DAPA collection endpoint. add `*` if querying from all collections" + in: "path" + required: true + schema: + type: "string" + - name: "limit" + description: "How many items in 1 page. defaulted to 10" + in: "query" + required: false + schema: + type: "string" + - name: "offset" + description: "How many items to skip. if limit == 10, offset = 10 for 2nd page, so on and so forth" + in: "query" + required: false + schema: + type: "string" + responses: + '200': + description: 'Successful query' + content: + application/json: + schema: + type: object + properties: + features: + description: 'STAC granule (aka item) array' + type: array + items: + type: object + '500': + description: 'Server Error' + '403': + description: 'Authorization Error' + '401': + description: 'Authorization Error' diff --git a/setup.py b/setup.py index e6091bfd..0c63f6f9 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ install_requires = [ 'fastjsonschema', 'xmltodict', - 'requests===2.27.1' + 'requests' ] flask_requires = [ @@ -16,16 +16,16 @@ setup( name="cumulus_lambda_functions", - version="1.6.19", + version="1.7.0", packages=find_packages(), install_requires=install_requires, - tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage'], + tests_require=['mock', 'nose', 'sphinx', 'sphinx_rtd_theme', 'coverage', 'pystac', 'python-dotenv', 'jsonschema'], test_suite='nose.collector', author=['Wai Phyo'], author_email=['wai.phyo@jpl.nasa.gov'], license='NONE', include_package_data=True, - python_requires="==3.7", + python_requires="==3.9", entry_points={ } ) diff --git a/tests/cumulus_lambda_functions/cumulus_download_granules/__init__.py b/tests/cumulus_lambda_functions/cumulus_download_granules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/cumulus_download_granules/test_download_granules.py b/tests/cumulus_lambda_functions/cumulus_download_granules/test_download_granules.py new file mode 100644 index 00000000..05de8ac5 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_download_granules/test_download_granules.py @@ -0,0 +1,31 @@ +import os +import tempfile +from glob import glob +from unittest import TestCase + +from cumulus_lambda_functions.cumulus_download_granules.download_granules import DownloadGranules + + +class TestDownloadGranules(TestCase): + def test_01(self): + os.environ['DAPA_API'] = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '7a1fglm2d54eoggj13lccivp25' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['COLLECTION_ID'] = 'SNDR_SNPP_ATMS_L1A_NGA___1' + os.environ['DOWNLOAD_DIR'] = '/etc/granules' + os.environ['VERIFY_SSL'] = 'FALSE' + os.environ['LIMITS'] = '100' + os.environ['LOG_LEVEL'] = '20' + os.environ['DATE_FROM'] = '2016-01-14T10:00:00.000Z' + os.environ['DATE_TO'] = '2016-01-15T10:06:00.000Z' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['DOWNLOAD_DIR'] = tmp_dir_name + DownloadGranules().start() + raw_files = glob(f'{tmp_dir_name}/*', recursive=True) + self.assertEqual(2, len(raw_files), f'wrong file count: {raw_files}') + return diff --git a/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py b/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py index 151991af..9968ea66 100644 --- a/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py +++ b/tests/cumulus_lambda_functions/cumulus_stac/test_collection_transformer.py @@ -1,68 +1,61 @@ import json from unittest import TestCase -from cumulus_lambda_functions.cumulus_stac.collection_transformer import STAC_COLLECTION_SCHEMA +import jsonschema + +from cumulus_lambda_functions.cumulus_stac.collection_transformer import STAC_COLLECTION_SCHEMA, CollectionTransformer from cumulus_lambda_functions.lib.json_validator import JsonValidator class TestItemTransformer(TestCase): def test_01(self): stac_validator = JsonValidator(json.loads(STAC_COLLECTION_SCHEMA)) - source = '''{ - "published": false, - "endingDateTime": "2016-01-31T19:59:59.991043", - "status": "completed", - "timestamp": 1648050501578, - "createdAt": 1648050499079, - "processingEndDateTime": "2022-03-23T15:48:20.869Z", - "productVolume": 18096656, - "timeToPreprocess": 20.302, - "timeToArchive": 0, - "productionDateTime": "2016-02-01T02:45:59.639000Z", - "execution": "https://console.aws.amazon.com/states/home?region=us-west-2#/executions/details/arn:aws:states:us-west-2:884500545225:execution:am-uds-dev-cumulus-IngestGranule:ec602ca7-0243-44df-adc0-28fb8a486d54", + source = { + "createdAt": 1647992847582, + "granuleId": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$", + "process": "modis", + "dateFrom": "1990-01-01T00:00:00Z", + "dateTo": "1991-01-01T00:00:00Z", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "name": "ATMS_SCIENCE_Group", "files": [ { - "bucket": "am-uds-dev-cumulus-internal", - "key": "ATMS_SCIENCE_Group___1/P1570515ATMSSCIENCEAAT16032024518500.PDS", - "size": 760, - "fileName": "P1570515ATMSSCIENCEAAT16032024518500.PDS", - "source": "data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2016/031//P1570515ATMSSCIENCEAAT16032024518500.PDS", - "type": "data" + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS", + "type": "data", + "reportToEms": True }, { - "bucket": "am-uds-dev-cumulus-internal", - "key": "ATMS_SCIENCE_Group___1/P1570515ATMSSCIENCEAAT16032024518501.PDS", - "size": 18084600, - "fileName": "P1570515ATMSSCIENCEAAT16032024518501.PDS", - "source": "data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2016/031//P1570515ATMSSCIENCEAAT16032024518501.PDS", + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS", + "reportToEms": True, "type": "metadata" }, { - "bucket": "am-uds-dev-cumulus-internal", - "key": "ATMS_SCIENCE_Group___1/P1570515ATMSSCIENCEAAT16032024518501.PDS.xml", - "size": 9547, - "fileName": "P1570515ATMSSCIENCEAAT16032024518501.PDS.xml", - "source": "data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2016/031//P1570515ATMSSCIENCEAAT16032024518501.PDS.xml", + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000001.PDS.xml", + "reportToEms": True, "type": "metadata" }, { - "bucket": "am-uds-dev-cumulus-internal", - "key": "ATMS_SCIENCE_Group___1/P1570515ATMSSCIENCEAAT16032024518500.PDS.cmr.xml", - "size": 1749, - "fileName": "P1570515ATMSSCIENCEAAT16032024518500.PDS.cmr.xml", + "bucket": "internal", + "regex": "^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00.PDS.cmr.xml$", + "sampleFileName": "P1570515ATMSSCIENCEAXT11344000000000.PDS.cmr.xml", + "reportToEms": True, "type": "metadata" } ], - "processingStartDateTime": "2022-03-23T15:45:03.732Z", - "updatedAt": 1648050501578, - "beginningDateTime": "2016-01-31T18:00:00.009057", - "provider": "snpp_provider_03", - "granuleId": "P1570515ATMSSCIENCEAAT16032024518500.PDS", - "collectionId": "ATMS_SCIENCE_Group___001", - "duration": 197.993, - "error": {}, - "lastUpdateDateTime": "2018-04-25T21:45:45.524053" - }''' + "granuleIdExtraction": "(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+", + "reportToEms": True, + "version": "001", + "duplicateHandling": "replace", + "updatedAt": 1647992847582, + "url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}", + "timestamp": 1647992849273 + } raw = { "type": "Collection", "stac_version": "1.0.0", @@ -85,9 +78,10 @@ def test_01(self): "links": [ { "rel": "root", - "href": ".", + "href": "./collection.json", }, ] } + raw = CollectionTransformer().to_stac(source) self.assertEqual(None, stac_validator.validate(raw), f'invalid stac format: {stac_validator}') return diff --git a/tests/cumulus_lambda_functions/cumulus_upload_granules/test_upload_granules.py b/tests/cumulus_lambda_functions/cumulus_upload_granules/test_upload_granules.py new file mode 100644 index 00000000..5c51cb62 --- /dev/null +++ b/tests/cumulus_lambda_functions/cumulus_upload_granules/test_upload_granules.py @@ -0,0 +1,125 @@ +import unittest +import os +import tempfile + +from cumulus_lambda_functions.cumulus_upload_granules.upload_granules import UploadGranules + + +class TestLUploadGranules(unittest.TestCase): + def test_01(self): + os.environ['DAPA_API'] = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev' + os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' + os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' + os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' + os.environ['CLIENT_ID'] = '7a1fglm2d54eoggj13lccivp25' + os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' + + os.environ['COLLECTION_ID'] = 'SNDR_SNPP_ATMS_L1A_NGA___1' + os.environ['PROVIDER_ID'] = 'SNPP' + os.environ['UPLOAD_DIR'] = '/tmp/snpp_upload_test_1' + os.environ['STAGING_BUCKET'] = 'am-uds-dev-cumulus-staging' + os.environ['VERIFY_SSL'] = 'false' + os.environ['DELETE_FILES'] = 'false' + + with tempfile.TemporaryDirectory() as tmp_dir_name: + os.environ['UPLOAD_DIR'] = tmp_dir_name + with open(os.path.join(tmp_dir_name, 'SNDR.SNPP.ATMS.L1A.nominal2.02.nc'), 'w') as ff: + ff.write('sample_file') + with open(os.path.join(tmp_dir_name, 'SNDR.SNPP.ATMS.L1A.nominal2.02.nc.cas'), 'w') as ff: + ff.write(''' + + + AggregateDir + snppatmsl1a + + + AutomaticQualityFlag + Passed + + + BuildId + v01.43.00 + + + CollectionLabel + L1AMw_nominal2 + + + DataGroup + sndr + + + EndDateTime + 2016-01-14T10:06:00.000Z + + + EndTAI93 + 726919569.000 + + + FileFormat + nc4 + + + FileLocation + /pge/out + + + Filename + SNDR.SNPP.ATMS.L1A.nominal2.02.nc + + + GranuleNumber + 101 + + + JobId + f163835c-9945-472f-bee2-2bc12673569f + + + ModelId + urn:npp:SnppAtmsL1a + + + NominalDate + 2016-01-14 + + + ProductName + SNDR.SNPP.ATMS.20160114T1000.m06.g101.L1A.L1AMw_nominal2.v03_15_00.D.201214135000.nc + + + ProductType + SNDR_SNPP_ATMS_L1A + + + ProductionDateTime + 2020-12-14T13:50:00.000Z + + + ProductionLocation + Sounder SIPS: JPL/Caltech (Dev) + + + ProductionLocationCode + D + + + RequestId + 1215 + + + StartDateTime + 2016-01-14T10:00:00.000Z + + + StartTAI93 + 726919209.000 + + + TaskId + 8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd + +''') + self.assertEqual('REGISTERED', UploadGranules().start().upper().strip(), 'wrong registration result') + return diff --git a/tests/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py b/tests/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py deleted file mode 100644 index 96fe5a32..00000000 --- a/tests/cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - - -os.environ['DAPA_API'] = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev' -os.environ['USERNAME'] = '/unity/uds/user/wphyo/username' -os.environ['PASSWORD'] = '/unity/uds/user/wphyo/dwssap' -os.environ['PASSWORD_TYPE'] = 'PARAM_STORE' -os.environ['CLIENT_ID'] = '7a1fglm2d54eoggj13lccivp25' -os.environ['COGNITO_URL'] = 'https://cognito-idp.us-west-2.amazonaws.com' - -os.environ['COLLECTION_ID'] = 'SNDR_SNPP_ATMS_L1A_NGA___1' -os.environ['PROVIDER_ID'] = 'SNPP' -os.environ['UPLOAD_DIR'] = '/tmp/snpp_upload_test_1' -os.environ['STAGING_BUCKET'] = 'am-uds-dev-cumulus-staging' -os.environ['VERIFY_SSL'] = 'false' -os.environ['DELETE_FILES'] = 'false' - -from cumulus_lambda_functions.cumulus_upload_granules.upload_granules import UploadGranules -print(UploadGranules().start()) \ No newline at end of file diff --git a/tests/cumulus_lambda_functions/lib/cognito_login/test_cognito_token_retriever.py b/tests/cumulus_lambda_functions/lib/cognito_login/test_cognito_token_retriever.py new file mode 100644 index 00000000..80f61d7c --- /dev/null +++ b/tests/cumulus_lambda_functions/lib/cognito_login/test_cognito_token_retriever.py @@ -0,0 +1,27 @@ +import os +from unittest import TestCase + +from cumulus_lambda_functions.lib.cognito_login.cognito_token_retriever import CognitoTokenRetriever +from cumulus_lambda_functions.lib.constants import Constants + + +class TestCognitoTokenRetriever(TestCase): + def test_param_store_01(self): + # os.environ['AWS_ACCESS_KEY_ID'] = 'dd' + # os.environ['AWS_SECRET_ACCESS_KEY'] = 'ddd' + # os.environ['AWS_SESSION_TOKEN'] = 'ddd' + # os.environ['AWS_REGION'] = 'us-west-2' + + + os.environ[Constants.USERNAME] = '/unity/uds/user/wphyo/username' + os.environ[Constants.PASSWORD] = '/unity/uds/user/wphyo/dwssap' + # os.environ[Constants.USERNAME] = 'usps_username' + # os.environ[Constants.PASSWORD] = 'usps_password' + os.environ[Constants.PASSWORD_TYPE] = Constants.PARAM_STORE + os.environ[Constants.CLIENT_ID] = '7a1fglm2d54eoggj13lccivp25' # JPL Cloud + os.environ[Constants.CLIENT_ID] = '6ir9qveln397i0inh9pmsabq1' # MCP Test Cloud + + os.environ[Constants.COGNITO_URL] = 'https://cognito-idp.us-west-2.amazonaws.com' + result = CognitoTokenRetriever().start() + self.assertTrue(len(result) > 0, 'empty token') + return diff --git a/tests/cumulus_lambda_functions/lib/utils/__init__.py b/tests/cumulus_lambda_functions/lib/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/lib/utils/test_lambda_api_gateway_utils.py b/tests/cumulus_lambda_functions/lib/utils/test_lambda_api_gateway_utils.py new file mode 100644 index 00000000..a7ac42b6 --- /dev/null +++ b/tests/cumulus_lambda_functions/lib/utils/test_lambda_api_gateway_utils.py @@ -0,0 +1,87 @@ +import unittest + +from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils + + +class TestCognitoLogin(unittest.TestCase): + def test_01(self): + sample_event = { + 'resource': '/am-uds-dapa/collections/{collectionId}/items', + 'path': '/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items', 'httpMethod': 'GET', + 'headers': {'accept': '*/*', + 'Authorization': 'Bearer eyJraWQiOiJzdE42WWl0eGxWZmJnY1ByRnJLWVQ1MEdjVWRIZWNBaWFKQ09peUxLVHNZPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiIwMGRkYTdmNy1mNjE4LTRmNDMtYWFmNC1iYmM4YmExNDc2ODAiLCJjb2duaXRvOmdyb3VwcyI6WyJVbml0eV9WaWV3ZXIiLCJVbml0eV9BZG1pbiJdLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtd2VzdC0yLmFtYXpvbmF3cy5jb21cL3VzLXdlc3QtMl9GTER5WEUybU8iLCJjbGllbnRfaWQiOiI3YTFmZ2xtMmQ1NGVvZ2dqMTNsY2NpdnAyNSIsIm9yaWdpbl9qdGkiOiI3MDcyZjQ2NC1mMWFjLTQzYjMtOTQ5Yy1iM2JjZTA1YWExOGEiLCJldmVudF9pZCI6IjUxMjk3NTA3LTdkMGMtNDcyYi1hNjM1LTY4YWJmNzJiZTZlMiIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYXdzLmNvZ25pdG8uc2lnbmluLnVzZXIuYWRtaW4iLCJhdXRoX3RpbWUiOjE2NTk3MTUxNzMsImV4cCI6MTY1OTcxODc3MywiaWF0IjoxNjU5NzE1MTczLCJqdGkiOiJiZmU2ZWY2OS1mNTc0LTQ1OTMtYjFlZi1iYmRhNzQ0NjFkMTciLCJ1c2VybmFtZSI6IndwaHlvIn0.X1K9NeZ661nCbD4PoNe_ZsjZITrs_OEzQ0ZjbsYGAQXCFwQmCaGiHhj0klb9xs8ByJ4VG7il8p_nu96QG0nkuv-HQXjG6YDxcA72rIvxK7w4LVF6_SxacDLHGsr3669Ptz6mzG5ql5AhDgiwCAYVgvlDyqtMNrbOEIe6HZDz6Hn12DUkk_7pgbZA77mtzikDrsSdJlT3RknuAooeZZGRwqnLsRZ7l4dw25uhPKjYDBSF6Psbc9IfzqvE-rTnloQ5atS6E5XL_Ig4YACBVDMqVqfiX11Uj5hxe2oI3t7o9JeePSwfwG-Z13zeI0XeldkaZW2E30qAzpob3EeQzuw_OQ', + 'Host': 'k3a3qmarxh.execute-api.us-west-2.amazonaws.com', 'User-Agent': 'curl/7.64.1', + 'X-Amzn-Trace-Id': 'Root=1-62ed40e1-6302916f16a2262c4f6012cf', + 'X-Forwarded-For': '128.149.247.57', + 'X-Forwarded-Port': '443', 'X-Forwarded-Proto': 'https'}, + 'multiValueHeaders': {'accept': ['*/*'], + 'Authorization': [ + 'Bearer eyJraWQiOiJzdE42WWl0eGxWZmJnY1ByRnJLWVQ1MEdjVWRIZWNBaWFKQ09peUxLVHNZPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiIwMGRkYTdmNy1mNjE4LTRmNDMtYWFmNC1iYmM4YmExNDc2ODAiLCJjb2duaXRvOmdyb3VwcyI6WyJVbml0eV9WaWV3ZXIiLCJVbml0eV9BZG1pbiJdLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAudXMtd2VzdC0yLmFtYXpvbmF3cy5jb21cL3VzLXdlc3QtMl9GTER5WEUybU8iLCJjbGllbnRfaWQiOiI3YTFmZ2xtMmQ1NGVvZ2dqMTNsY2NpdnAyNSIsIm9yaWdpbl9qdGkiOiI3MDcyZjQ2NC1mMWFjLTQzYjMtOTQ5Yy1iM2JjZTA1YWExOGEiLCJldmVudF9pZCI6IjUxMjk3NTA3LTdkMGMtNDcyYi1hNjM1LTY4YWJmNzJiZTZlMiIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYXdzLmNvZ25pdG8uc2lnbmluLnVzZXIuYWRtaW4iLCJhdXRoX3RpbWUiOjE2NTk3MTUxNzMsImV4cCI6MTY1OTcxODc3MywiaWF0IjoxNjU5NzE1MTczLCJqdGkiOiJiZmU2ZWY2OS1mNTc0LTQ1OTMtYjFlZi1iYmRhNzQ0NjFkMTciLCJ1c2VybmFtZSI6IndwaHlvIn0.X1K9NeZ661nCbD4PoNe_ZsjZITrs_OEzQ0ZjbsYGAQXCFwQmCaGiHhj0klb9xs8ByJ4VG7il8p_nu96QG0nkuv-HQXjG6YDxcA72rIvxK7w4LVF6_SxacDLHGsr3669Ptz6mzG5ql5AhDgiwCAYVgvlDyqtMNrbOEIe6HZDz6Hn12DUkk_7pgbZA77mtzikDrsSdJlT3RknuAooeZZGRwqnLsRZ7l4dw25uhPKjYDBSF6Psbc9IfzqvE-rTnloQ5atS6E5XL_Ig4YACBVDMqVqfiX11Uj5hxe2oI3t7o9JeePSwfwG-Z13zeI0XeldkaZW2E30qAzpob3EeQzuw_OQ'], + 'Host': [ + 'k3a3qmarxh.execute-api.us-west-2.amazonaws.com'], + 'User-Agent': [ + 'curl/7.64.1'], + 'X-Amzn-Trace-Id': [ + 'Root=1-62ed40e1-6302916f16a2262c4f6012cf'], + 'X-Forwarded-For': [ + '128.149.247.57'], + 'X-Forwarded-Port': [ + '443'], + 'X-Forwarded-Proto': [ + 'https']}, + 'queryStringParameters': {'datetime': '1990-01-01T00:00:00Z/2021-01-03T00:00:00Z'}, + 'multiValueQueryStringParameters': {'datetime': ['1990-01-01T00:00:00Z/2021-01-03T00:00:00Z']}, + 'pathParameters': {'collectionId': 'L0_SNPP_ATMS_SCIENCE___1'}, 'stageVariables': {'VPCLINK': 'czcxgk'}, + 'requestContext': {'resourceId': 'pm8zuj', + 'authorizer': {'numberKey': '123', 'booleanKey': 'true', 'stringKey': 'stringval', + 'principalId': 'user', 'integrationLatency': 70}, + 'resourcePath': '/am-uds-dapa/collections/{collectionId}/items', 'httpMethod': 'GET', + 'extendedRequestId': 'WZcTREQqvHcFztw=', 'requestTime': '05/Aug/2022:16:10:09 +0000', + 'path': '/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items', + 'accountId': '884500545225', 'protocol': 'HTTP/1.1', 'stage': 'dev', + 'domainPrefix': 'k3a3qmarxh', 'requestTimeEpoch': 1659715809560, + 'requestId': '240ae2fb-c5cc-4e83-9040-d44ed127c889', + 'identity': {'cognitoIdentityPoolId': None, 'accountId': None, 'cognitoIdentityId': None, + 'caller': None, 'sourceIp': '128.149.247.57', 'principalOrgId': None, + 'accessKey': None, 'cognitoAuthenticationType': None, + 'cognitoAuthenticationProvider': None, 'userArn': None, + 'userAgent': 'curl/7.64.1', + 'user': None}, + 'domainName': 'k3a3qmarxh.execute-api.us-west-2.amazonaws.com', + 'apiId': 'k3a3qmarxh'}, + 'body': None, + 'isBase64Encoded': False + } + lambda_pagination = LambdaApiGatewayUtils(sample_event, 10) + links = lambda_pagination.generate_pagination_links() + links_dict = {k['rel']: k['href'] for k in links} + self.assertTrue('next' in links_dict, f'next missing in {links}') + self.assertTrue('prev' in links_dict, f'prev missing in {links}') + self.assertTrue('self' in links_dict, f'self missing in {links}') + self.assertTrue('root' in links_dict, f'root missing in {links}') + next_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=10&offset=10' + self.assertEqual(sorted(next_url), sorted(links_dict['next']), f'wrong next url. {next_url} vs {links_dict["next"]}') + prev_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=10&offset=0' + self.assertEqual(sorted(prev_url), sorted(links_dict['prev']), f'wrong next url. {prev_url} vs {links_dict["prev"]}') + current_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=10&offset=0' + self.assertEqual(sorted(current_url), sorted(links_dict['self']), f'wrong self url. {current_url} vs {links_dict["self"]}') + + lambda_pagination = LambdaApiGatewayUtils(sample_event, 0) + links = lambda_pagination.generate_pagination_links() + links_dict = {k['rel']: k['href'] for k in links} + self.assertEqual('', links_dict['next'], f'wrong next empty url. {links_dict["next"]}') + self.assertEqual('', links_dict['prev'], f'wrong next empty url. {links_dict["prev"]}') + current_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=0&offset=0' + self.assertEqual(sorted(current_url), sorted(links_dict['self']), f'wrong self url. {current_url} vs {links_dict["self"]}') + + sample_event['queryStringParameters']['offset'] = 10 + lambda_pagination = LambdaApiGatewayUtils(sample_event, 5) + links = lambda_pagination.generate_pagination_links() + links_dict = {k['rel']: k['href'] for k in links} + next_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=5&offset=15' + self.assertEqual(sorted(next_url), sorted(links_dict['next']), f'wrong next url. {next_url} vs {links_dict["next"]}') + prev_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=5&offset=5' + self.assertEqual(sorted(prev_url), sorted(links_dict['prev']), f'wrong next url. {prev_url} vs {links_dict["prev"]}') + current_url = 'https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev/am-uds-dapa/collections/L0_SNPP_ATMS_SCIENCE___1/items?datetime=1990-01-01T00:00:00Z/2021-01-03T00:00:00Z&limit=5&offset=10' + self.assertEqual(sorted(current_url), sorted(links_dict['self']), f'wrong self url. {current_url} vs {links_dict["self"]}') + return diff --git a/tests/integration_tests/.env.tpl b/tests/integration_tests/.env.tpl new file mode 100644 index 00000000..4247048b --- /dev/null +++ b/tests/integration_tests/.env.tpl @@ -0,0 +1,5 @@ +USERNAME=base64-encoded-str +PASSWORD=base64-encoded-str +CLIENT_ID=7a1fglm2d54eoggj13lccivp25 +COGNITO_URL=https://cognito-idp.us-west-2.amazonaws.com +UNITY_URL=https://k3a3qmarxh.execute-api.us-west-2.amazonaws.com/dev diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration_tests/test_dapa_stac.py b/tests/integration_tests/test_dapa_stac.py new file mode 100644 index 00000000..272d79b5 --- /dev/null +++ b/tests/integration_tests/test_dapa_stac.py @@ -0,0 +1,46 @@ +import base64 +import json +import os +from unittest import TestCase + +import pystac +import requests +from dotenv import load_dotenv + +from cumulus_lambda_functions.lib.cognito_login.cognito_login import CognitoLogin + + +class TestDapaStac(TestCase): + def test_collection_01(self): + load_dotenv() + cognito_login = CognitoLogin()\ + .with_client_id(os.environ.get('CLIENT_ID', ''))\ + .with_cognito_url(os.environ.get('COGNITO_URL', ''))\ + .with_verify_ssl(False)\ + .start(base64.standard_b64decode(os.environ.get('USERNAME')).decode(), base64.standard_b64decode(os.environ.get('PASSWORD')).decode()) + collection_url = f'{os.environ.get("UNITY_URL")}/am-uds-dapa/collections' + response = requests.get(url=collection_url, headers={'Authorization': f'Bearer {cognito_login.token}'}, verify=False) + self.assertEqual(response.status_code, 200, 'wrong status code') + response_json = json.loads(response.content.decode()) + self.assertTrue(len(response_json['features']) > 0, f'empty collections. Need collections to compare') + for each_feature in response_json['features']: + validation_result = pystac.Collection.from_dict(each_feature).validate() + self.assertTrue(isinstance(validation_result, list), f'wrong validation for : {json.dumps(each_feature, indent=4)}. details: {validation_result}') + return + + def test_granules_01(self): + load_dotenv() + cognito_login = CognitoLogin()\ + .with_client_id(os.environ.get('CLIENT_ID', ''))\ + .with_cognito_url(os.environ.get('COGNITO_URL', ''))\ + .with_verify_ssl(False)\ + .start(base64.standard_b64decode(os.environ.get('USERNAME')).decode(), base64.standard_b64decode(os.environ.get('PASSWORD')).decode()) + collection_url = f'{os.environ.get("UNITY_URL")}/am-uds-dapa/collections/*/items' + response = requests.get(url=collection_url, headers={'Authorization': f'Bearer {cognito_login.token}'}, verify=False) + self.assertEqual(response.status_code, 200, 'wrong status code') + response_json = json.loads(response.content.decode()) + self.assertTrue(len(response_json['features']) > 0, f'empty granules. Need collections to compare') + for each_feature in response_json['features']: + validation_result = pystac.Item.from_dict(each_feature).validate() + self.assertTrue(isinstance(validation_result, list), f'wrong validation for : {json.dumps(each_feature, indent=4)}. details: {validation_result}') + return