From 33475e0f9c67945680c930f3e4f22cd26029a2fb Mon Sep 17 00:00:00 2001 From: wphyojpl <38299756+wphyojpl@users.noreply.github.com> Date: Mon, 9 Dec 2024 05:10:50 -0800 Subject: [PATCH 01/10] fix: index to ES is not setting bbox correctly (#475) * breaking: using latest uds-lib + update docker * feat: use latest uds-lib * fix: index to ES is not setting bbox correctly --- .../granules_to_es/granules_indexer.py | 4 +- .../stac_input_metadata.py | 15 + .../granules_to_es/test_granules_indexer.py | 315 ++++++++++-------- 3 files changed, 195 insertions(+), 139 deletions(-) diff --git a/cumulus_lambda_functions/granules_to_es/granules_indexer.py b/cumulus_lambda_functions/granules_to_es/granules_indexer.py index 853cc096..e9db5b7d 100644 --- a/cumulus_lambda_functions/granules_to_es/granules_indexer.py +++ b/cumulus_lambda_functions/granules_to_es/granules_indexer.py @@ -47,7 +47,7 @@ def __get_potential_files(self): potential_files = [] self.__input_file_list = self.__cumulus_record['files'] for each_file in self.__input_file_list: - if 'type' in each_file and each_file['type'].strip().lower() != self.__valid_filetype_name: + if 'type' in each_file and self.__valid_filetype_name not in each_file['type'].strip().lower(): LOGGER.debug(f'Not metadata. skipping {each_file}') continue if 'fileName' not in each_file and 'name' in each_file: # add fileName if there is only name @@ -93,6 +93,8 @@ def start(self): else: LOGGER.warning(f'unable to find STAC JSON file in {potential_files}') stac_item = ItemTransformer().to_stac(self.__cumulus_record) + if stac_input_meta is not None and stac_input_meta.bbox is not None: + stac_item['bbox'] = stac_input_meta.bbox if 'bbox' in stac_item: stac_item['bbox'] = GranulesDbIndex.to_es_bbox(stac_item['bbox']) collection_identifier = UdsCollections.decode_identifier(self.__cumulus_record['collectionId']) diff --git a/cumulus_lambda_functions/metadata_stac_generate_cmr/stac_input_metadata.py b/cumulus_lambda_functions/metadata_stac_generate_cmr/stac_input_metadata.py index df58cf16..06686210 100644 --- a/cumulus_lambda_functions/metadata_stac_generate_cmr/stac_input_metadata.py +++ b/cumulus_lambda_functions/metadata_stac_generate_cmr/stac_input_metadata.py @@ -17,6 +17,20 @@ def __init__(self, input_stac_dict: dict): self.__prod_dt = None self.__insert_dt = None self.__custom_properties = {} + self.__bbox = None + + @property + def bbox(self): + return self.__bbox + + @bbox.setter + def bbox(self, val): + """ + :param val: + :return: None + """ + self.__bbox = val + return @property def custom_properties(self): @@ -132,6 +146,7 @@ def start(self) -> GranuleMetadataProps: stac_item: Item = ItemTransformer().from_stac(self.__input_stac_dict) self.__custom_properties = deepcopy(stac_item.properties) self.__remove_default_keys_in_custom_props() + self.__bbox = stac_item.bbox # self.__custom_properties['collection_id'] = stac_item.collection_id # TODO version is included # collection_led_granule_id = stac_item.id if stac_item.id.startswith(stac_item.collection_id) else f'{stac_item.collection_id}:{stac_item.id}' # self.__custom_properties['granule_id'] = collection_led_granule_id # This needs to be start with collection_id to be consistent with cumulus granule_id which starts with collection diff --git a/tests/cumulus_lambda_functions/granules_to_es/test_granules_indexer.py b/tests/cumulus_lambda_functions/granules_to_es/test_granules_indexer.py index 7f7844f9..7af740c9c 100644 --- a/tests/cumulus_lambda_functions/granules_to_es/test_granules_indexer.py +++ b/tests/cumulus_lambda_functions/granules_to_es/test_granules_indexer.py @@ -1,164 +1,203 @@ import json +import os +from time import sleep from unittest import TestCase +from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 +from mdps_ds_lib.lib.utils.time_utils import TimeUtils +from pystac import Item, Asset + from cumulus_lambda_functions.granules_to_es.granules_indexer import GranulesIndexer +from cumulus_lambda_functions.lib.uds_db.granules_db_index import GranulesDbIndex class TestGranulesIndexer(TestCase): def test_01(self): + os.environ['ES_URL'] = 'vpc-uds-sbx-cumulus-es-qk73x5h47jwmela5nbwjte4yzq.us-west-2.es.amazonaws.com' + os.environ['ES_PORT'] = '9200' + + s3 = AwsS3() + mock_stac_item = Item(id='URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01', + geometry={ + "type": "Point", + "coordinates": [0.0, 0.0] + }, + bbox=[0.0, -10.0, 10.0, 0.0], + datetime=TimeUtils().parse_from_unix(9876543210, True).get_datetime_obj(), + properties={ + "start_datetime": "2016-01-31T18:00:00.009057Z", + "end_datetime": "2016-01-31T19:59:59.991043Z", + "created": "2016-02-01T02:45:59.639000Z", + "updated": "2022-03-23T15:48:21.578000Z", + "datetime": "2022-03-23T15:48:19.079000Z", + }, + collection='URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417', + assets={ + 'data': Asset('test_file01.nc', title='main data'), + 'metadata': Asset('test_file01.cmr.xml', title='metadata cas'), + 'metadata__cas': Asset('test_file01.nc.cas', title='metadata cas'), + 'metadata__stac': Asset('test_file01.nc.stac.json', title='metadata stac'), + }) + s3.set_s3_url('s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json')\ + .upload_bytes(json.dumps(mock_stac_item.to_dict(False, False)).encode()) cumulus_msg = { - "event": "Update", - "record": { - "collectionId": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417", - "createdAt": 1699568555580, - "duration": 36.81, - "error": { - "Cause": "None", - "Error": "Unknown Error" - }, - "execution": "https://console.aws.amazon.com/states/home?region=us-west-2#/executions/details/arn:aws:states:us-west-2:237868187491:execution:uds-sbx-cumulus-CatalogGranule:3ae4c03e-dcd1-4d25-8b8a-b8c2a3c126ae", - "granuleId": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01", - "processingEndDateTime": "2023-11-09T22:22:45.944Z", - "processingStartDateTime": "2023-11-09T22:22:10.949Z", - "productVolume": "1765", - "provider": "unity", - "published": False, - "status": "completed", - "timestamp": 1699568567282, - "timeToArchive": 0, - "timeToPreprocess": 20.302, - "updatedAt": 1699568567282, - "files": [ - { - "bucket": "uds-sbx-cumulus-staging", - "checksum": "9817be382b87c48ebe482b9c47d1525a", - "checksumType": "md5", - "fileName": "test_file01.cmr.xml", - "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml", - "size": 1768, - "source": "s3://uds-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml", - "type": "metadata" - }, - { - "bucket": "uds-sbx-cumulus-staging", - "checksum": "unknown", - "checksumType": "md5", - "fileName": "test_file01.nc.stac.json", - "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json", - "size": -1, - "source": "s3://uds-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json", - "type": "metadata__stac" - }, - { - "bucket": "uds-sbx-cumulus-staging", - "checksum": "unknown", - "checksumType": "md5", - "fileName": "test_file01.nc.cas", - "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.cas", - "size": -1, - "source": "s3://uds-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.cas", - "type": "metadata__cas" - }, - { - "bucket": "uds-sbx-cumulus-staging", - "checksum": "unknown", - "checksumType": "md5", - "fileName": "test_file01.nc", - "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc", - "size": -1, - "source": "s3://uds-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc", - "type": "data" - } - ], - "beginningDateTime": "2016-01-31T18:00:00.009Z", - "endingDateTime": "2016-01-31T19:59:59.991Z", - "lastUpdateDateTime": "2018-04-25T21:45:45.524Z", - "productionDateTime": "1970-01-01T00:00:00.000Z", - "queryFields": { - "cnm": { - "product": { - "name": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01", - "files": [ + "event": "Update", + "record": { + "collectionId": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417", + "createdAt": 1699568555580, + "duration": 36.81, + "error": { + "Cause": "None", + "Error": "Unknown Error" + }, + "execution": "https://console.aws.amazon.com/states/home?region=us-west-2#/executions/details/arn:aws:states:us-west-2:237868187491:execution:uds-sbx-cumulus-CatalogGranule:3ae4c03e-dcd1-4d25-8b8a-b8c2a3c126ae", + "granuleId": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01", + "processingEndDateTime": "2023-11-09T22:22:45.944Z", + "processingStartDateTime": "2023-11-09T22:22:10.949Z", + "productVolume": "1765", + "provider": "unity", + "published": False, + "status": "completed", + "timestamp": 1699568567282, + "timeToArchive": 0, + "timeToPreprocess": 20.302, + "updatedAt": 1699568567282, + "files": [ { - "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc", - "name": "test_file01.nc", - "size": -1, - "type": "data", - "checksum": "unknown", - "checksumType": "md5" + "bucket": "uds-sbx-cumulus-staging", + "checksum": "9817be382b87c48ebe482b9c47d1525a", + "checksumType": "md5", + "fileName": "test_file01.cmr.xml", + "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml", + "size": 1768, + "source": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml", + "type": "metadata" }, { - "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.cas", - "name": "test_file01.nc.cas", - "size": -1, - "type": "metadata__cas", - "checksum": "unknown", - "checksumType": "md5" + "bucket": "uds-sbx-cumulus-staging", + "checksum": "unknown", + "checksumType": "md5", + "fileName": "test_file01.nc.stac.json", + "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json", + "size": -1, + "source": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json", + "type": "metadata__stac" }, { - "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json", - "name": "test_file01.nc.stac.json", - "size": -1, - "type": "metadata__stac", - "checksum": "unknown", - "checksumType": "md5" + "bucket": "uds-sbx-cumulus-staging", + "checksum": "unknown", + "checksumType": "md5", + "fileName": "test_file01.nc.cas", + "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.cas", + "size": -1, + "source": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.cas", + "type": "metadata__cas" }, { - "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml", - "name": "test_file01.cmr.xml", - "size": 1768, - "type": "metadata", - "checksum": "9817be382b87c48ebe482b9c47d1525a", - "checksumType": "md5" + "bucket": "uds-sbx-cumulus-staging", + "checksum": "unknown", + "checksumType": "md5", + "fileName": "test_file01.nc", + "key": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc", + "size": -1, + "source": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc", + "type": "data" } - ], - "dataVersion": "2311091417" - }, - "version": "1.6.0", - "provider": "unity", - "response": { - "status": "SUCCESS" - }, - "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION", - "identifier": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01", - "receivedTime": "2023-11-09T22:22:23.399Z", - "submissionTime": "2023-11-09T22:21:21.989722", - "processCompleteTime": "2023-11-09T22:22:41.220Z" - } + ], + "beginningDateTime": "2016-01-31T18:00:00.009Z", + "endingDateTime": "2016-01-31T19:59:59.991Z", + "lastUpdateDateTime": "2018-04-25T21:45:45.524Z", + "productionDateTime": "1970-01-01T00:00:00.000Z", + "queryFields": { + "cnm": { + "product": { + "name": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01", + "files": [ + { + "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc", + "name": "test_file01.nc", + "size": -1, + "type": "data", + "checksum": "unknown", + "checksumType": "md5" + }, + { + "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.cas", + "name": "test_file01.nc.cas", + "size": -1, + "type": "metadata__cas", + "checksum": "unknown", + "checksumType": "md5" + }, + { + "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.nc.stac.json", + "name": "test_file01.nc.stac.json", + "size": -1, + "type": "metadata__stac", + "checksum": "unknown", + "checksumType": "md5" + }, + { + "uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01/test_file01.cmr.xml", + "name": "test_file01.cmr.xml", + "size": 1768, + "type": "metadata", + "checksum": "9817be382b87c48ebe482b9c47d1525a", + "checksumType": "md5" + } + ], + "dataVersion": "2311091417" + }, + "version": "1.6.0", + "provider": "unity", + "response": { + "status": "SUCCESS" + }, + "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION", + "identifier": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:UDS_COLLECTION___2311091417:test_file01", + "receivedTime": "2023-11-09T22:22:23.399Z", + "submissionTime": "2023-11-09T22:21:21.989722", + "processCompleteTime": "2023-11-09T22:22:41.220Z" + } + } } - } } sns_msg = { - "Type" : "Notification", - "MessageId" : "f6441383-4a99-5f27-8a66-4e44177ea7f4", - "TopicArn" : "arn:aws:sns:us-west-2:237868187491:uds-sbx-cumulus-report-granules-topic", - "Message" : json.dumps(cumulus_msg), - "Timestamp" : "2023-11-09T22:22:31.148Z", - "SignatureVersion" : "1", - "Signature" : "Skm4aumaUGxZA76/Jdya+6a42805KvAn6PrZIwXdKHE+ng37e+aN75SuCTDrv5hzeRFxA8YSoEYMG+00CvnoVN3gtsVt/o78Nkj5lr2oMCwNj2k5kwyEve4BetRelyXF1BTc7ptD7MYsSVGrIWZQwqNqUviDfBdI1nxujDiZvWnjAPWjJA8+cjx2acFAbaTzIhN90V3Fn0yOtveVXblAUZQ3EwF8Cv0CsTJFVPYliguw72s2r+9xPbc5Yj8dBL4B38HI7JC+u6qL8vgzIh+/wVlpqOef5P23qFeYDE533318EUEDfrkRs//LCbe+lcoTzka5qwOWaveMbIM9tstmeg==", - "SigningCertURL" : "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-01d088a6f77103d0fe307c0069e40ed6.pem", - "UnsubscribeURL" : "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:237868187491:uds-sbx-cumulus-report-granules-topic:c003b5ee-09a7-4129-9873-a1629868e8bd" + "Type": "Notification", + "MessageId": "f6441383-4a99-5f27-8a66-4e44177ea7f4", + "TopicArn": "arn:aws:sns:us-west-2:237868187491:uds-sbx-cumulus-report-granules-topic", + "Message": json.dumps(cumulus_msg), + "Timestamp": "2023-11-09T22:22:31.148Z", + "SignatureVersion": "1", + "Signature": "Skm4aumaUGxZA76/Jdya+6a42805KvAn6PrZIwXdKHE+ng37e+aN75SuCTDrv5hzeRFxA8YSoEYMG+00CvnoVN3gtsVt/o78Nkj5lr2oMCwNj2k5kwyEve4BetRelyXF1BTc7ptD7MYsSVGrIWZQwqNqUviDfBdI1nxujDiZvWnjAPWjJA8+cjx2acFAbaTzIhN90V3Fn0yOtveVXblAUZQ3EwF8Cv0CsTJFVPYliguw72s2r+9xPbc5Yj8dBL4B38HI7JC+u6qL8vgzIh+/wVlpqOef5P23qFeYDE533318EUEDfrkRs//LCbe+lcoTzka5qwOWaveMbIM9tstmeg==", + "SigningCertURL": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-01d088a6f77103d0fe307c0069e40ed6.pem", + "UnsubscribeURL": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:237868187491:uds-sbx-cumulus-report-granules-topic:c003b5ee-09a7-4129-9873-a1629868e8bd" } sample_event = { - "Records": [ - { - "messageId": "6210f778-d081-4ae9-a861-8534d612dfae", - "receiptHandle": "AQEBk55DchogyQzpVsH1A4YEj4K/PcVuIG9Em/a6/4AHIA4G5vLPiHVElNiuMfYc1ussk2U//JwZbD788Fv8u6W22L3AJ1U8EIcGJ57aibpmd6tSCWLS5q5FA4u2X2Jq5z+lCX5NZXzNDYMqMJaCGtBkcYi4a9LDXtD+U7HWX0V8OPhFFF2a1qUu+E05c16f5OmE7wRJ3SFrRmtJOhp2DigKKsw6VJtZklTm6uILMOL1ETOTlbA02dhF16fjcXlAACirDp0Yo9pi91FrpEljOYkqAO9AX4WMbEjAPZrnaATfYmRqCTOlnrIK8xvgEPgIu/OOub7KBYh6AQn7U8QBNoASkXkn31dqyM2I+KosKy2VeJO9cjPTahhXtkW7zUFA6863Czt2oHqL6Rvwsjr+7TikfQ==", - "body": json.dumps(sns_msg), - "attributes": { - "ApproximateReceiveCount": "6", - "SentTimestamp": "1644255065441", - "SenderId": "AIDALVP5ID7KAVBU2CQ3O", - "ApproximateFirstReceiveTimestamp": "1644255065441" - }, - "messageAttributes": {}, - "md5OfBody": "00cb0a5ed122862537ab6115dae36f69", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws-us-gov:sqs:us-gov-west-1:440216117821:send_records_to_es", - "awsRegion": "us-gov-west-1" - } - ] + "Records": [ + { + "messageId": "6210f778-d081-4ae9-a861-8534d612dfae", + "receiptHandle": "AQEBk55DchogyQzpVsH1A4YEj4K/PcVuIG9Em/a6/4AHIA4G5vLPiHVElNiuMfYc1ussk2U//JwZbD788Fv8u6W22L3AJ1U8EIcGJ57aibpmd6tSCWLS5q5FA4u2X2Jq5z+lCX5NZXzNDYMqMJaCGtBkcYi4a9LDXtD+U7HWX0V8OPhFFF2a1qUu+E05c16f5OmE7wRJ3SFrRmtJOhp2DigKKsw6VJtZklTm6uILMOL1ETOTlbA02dhF16fjcXlAACirDp0Yo9pi91FrpEljOYkqAO9AX4WMbEjAPZrnaATfYmRqCTOlnrIK8xvgEPgIu/OOub7KBYh6AQn7U8QBNoASkXkn31dqyM2I+KosKy2VeJO9cjPTahhXtkW7zUFA6863Czt2oHqL6Rvwsjr+7TikfQ==", + "body": json.dumps(sns_msg), + "attributes": { + "ApproximateReceiveCount": "6", + "SentTimestamp": "1644255065441", + "SenderId": "AIDALVP5ID7KAVBU2CQ3O", + "ApproximateFirstReceiveTimestamp": "1644255065441" + }, + "messageAttributes": {}, + "md5OfBody": "00cb0a5ed122862537ab6115dae36f69", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws-us-gov:sqs:us-gov-west-1:440216117821:send_records_to_es", + "awsRegion": "us-gov-west-1" + } + ] } granules_indexer = GranulesIndexer(sample_event) granules_indexer.start() + granules_index = GranulesDbIndex() + sleep(3) + result = granules_index.get_entry('UDS_LOCAL_TEST', 'DEV', mock_stac_item.id) + print(result) + self.assertEqual(result['bbox'], {'type': 'envelope', 'coordinates': [[0.0, 0.0], [10.0, -10.0]]}, f'wrong bbox') return From f95b3f58b0819200bb24b8ef120b7260dd69a10b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 05:12:39 -0800 Subject: [PATCH 02/10] chore: update version + change log (#476) Co-authored-by: wphyojpl --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7d5cb6e..022d15de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.1] - 2024-12-09 +### Fixed +- [#475](https://github.com/unity-sds/unity-data-services/pull/475) fix: index to es is not setting bbox correctly + ## [9.1.0] - 2024-12-03 ### Changed - [#472](https://github.com/unity-sds/unity-data-services/pull/472) feat: amalgamation download type diff --git a/setup.py b/setup.py index a0104f61..e092e45a 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="cumulus_lambda_functions", - version="9.1.0", + version="9.1.1", packages=find_packages(), install_requires=install_requires, package_data={ From 4d0de302b3e19b8982722f98db81479259896408 Mon Sep 17 00:00:00 2001 From: wphyojpl <38299756+wphyojpl@users.noreply.github.com> Date: Mon, 9 Dec 2024 05:12:55 -0800 Subject: [PATCH 03/10] fix: Wrong location archive keys (#448) * feat: removed duplicated classes + tests * feat: update setup.py with the library * feat: bump version in requirements.txt * feat: update import statements * fix: update test cases * fix: removing boto3 in lambda zip * fix: bump uds lib + make checksum + size valid for empty str + remove duplicates * fix: need to fall back to semi-brute force to avoid duplication * fix: wrong location for archive keys * fix: update test * fix: remove unused file --- .../uds_api/dapa/granules_dapa_query_es.py | 27 ++-- .../uds_api/__init__.py | 0 .../uds_api/dapa/__init__.py | 0 .../dapa/test_granules_dapa_query_es.py | 142 ++++++++++++++++++ tests/integration_tests/test_uds_api.py | 2 +- 5 files changed, 159 insertions(+), 12 deletions(-) create mode 100644 tests/cumulus_lambda_functions/uds_api/__init__.py create mode 100644 tests/cumulus_lambda_functions/uds_api/dapa/__init__.py create mode 100644 tests/cumulus_lambda_functions/uds_api/dapa/test_granules_dapa_query_es.py diff --git a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py index a4cf9873..913c2f02 100644 --- a/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py +++ b/cumulus_lambda_functions/uds_api/dapa/granules_dapa_query_es.py @@ -75,6 +75,8 @@ def __get_time_range_terms(self): ] def __create_pagination_links(self, page_marker_str): + if self.__pagination_link_obj is None: + return [] new_queries = deepcopy(self.__pagination_link_obj.org_query_params) new_queries['limit'] = int(new_queries['limit'] if 'limit' in new_queries else self.__limit) current_page = f"{self.__pagination_link_obj.requesting_base_url}?{'&'.join([f'{k}={v}' for k, v in new_queries.items()])}" @@ -114,8 +116,20 @@ def archive_single_granule(self, granule_id): daac_archiver.send_to_daac_internal(cnm_response) return + def __restructure_each_granule_result(self, each_granules_query_result_stripped): + if 'event_time' in each_granules_query_result_stripped: + each_granules_query_result_stripped.pop('event_time') + if 'bbox' in each_granules_query_result_stripped: + each_granules_query_result_stripped['bbox'] = GranulesDbIndex.from_es_bbox(each_granules_query_result_stripped['bbox']) + for each_archiving_key in GranulesIndexMapping.archiving_keys: + if each_archiving_key in each_granules_query_result_stripped: + each_granules_query_result_stripped['properties'][each_archiving_key] = each_granules_query_result_stripped.pop(each_archiving_key) + return + def get_single_granule(self, granule_id): granules_query_dsl = { + 'size': 1, + 'sort': [{'id': {'order': 'asc'}}], 'query': {'bool': {'must': [{ 'term': {'id': granule_id} }]}} @@ -132,10 +146,7 @@ def get_single_granule(self, granule_id): each_granules_query_result_stripped = granules_query_result['hits']['hits'][0]['_source'] self_link = Link(rel='self', target=f'{self.__base_url}/{WebServiceConstants.COLLECTIONS}/{self.__collection_id}/items/{each_granules_query_result_stripped["id"]}', media_type='application/json', title=each_granules_query_result_stripped["id"]).to_dict(False) each_granules_query_result_stripped['links'].append(self_link) - if 'event_time' in each_granules_query_result_stripped: - each_granules_query_result_stripped.pop('event_time') - if 'bbox' in each_granules_query_result_stripped: - each_granules_query_result_stripped['bbox'] = GranulesDbIndex.from_es_bbox(each_granules_query_result_stripped['bbox']) + self.__restructure_each_granule_result(each_granules_query_result_stripped) return each_granules_query_result_stripped def start(self): @@ -152,13 +163,7 @@ def start(self): for each_granules_query_result_stripped in granules_query_result_stripped: self_link = Link(rel='self', target=f'{self.__base_url}/{WebServiceConstants.COLLECTIONS}/{self.__collection_id}/items/{each_granules_query_result_stripped["id"]}', media_type='application/json', title=each_granules_query_result_stripped["id"]).to_dict(False) each_granules_query_result_stripped['links'].append(self_link) - if 'event_time' in each_granules_query_result_stripped: - each_granules_query_result_stripped.pop('event_time') - if 'bbox' in each_granules_query_result_stripped: - each_granules_query_result_stripped['bbox'] = GranulesDbIndex.from_es_bbox(each_granules_query_result_stripped['bbox']) - for each_archiving_key in GranulesIndexMapping.archiving_keys: - if each_archiving_key in each_granules_query_result_stripped: - each_granules_query_result_stripped['properties'][each_archiving_key] = each_granules_query_result_stripped.pop(each_archiving_key) + self.__restructure_each_granule_result(each_granules_query_result_stripped) pagination_link = '' if len(granules_query_result['hits']['hits']) < self.__limit else ','.join(granules_query_result['hits']['hits'][-1]['sort']) return { 'statusCode': 200, diff --git a/tests/cumulus_lambda_functions/uds_api/__init__.py b/tests/cumulus_lambda_functions/uds_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/uds_api/dapa/__init__.py b/tests/cumulus_lambda_functions/uds_api/dapa/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/uds_api/dapa/test_granules_dapa_query_es.py b/tests/cumulus_lambda_functions/uds_api/dapa/test_granules_dapa_query_es.py new file mode 100644 index 00000000..9cc3fdc4 --- /dev/null +++ b/tests/cumulus_lambda_functions/uds_api/dapa/test_granules_dapa_query_es.py @@ -0,0 +1,142 @@ +import os +from unittest import TestCase + +from cumulus_lambda_functions.lib.uds_db.granules_db_index import GranulesDbIndex +from cumulus_lambda_functions.uds_api.dapa.granules_dapa_query_es import GranulesDapaQueryEs + + +class TestGranulesDapaQueryEs(TestCase): + def setUp(self) -> None: + super().setUp() + self.custom_metadata_body = { + 'tag': {'type': 'keyword'}, + 'c_data1': {'type': 'long'}, + 'c_data2': {'type': 'boolean'}, + 'c_data3': {'type': 'keyword'}, + } + + self.tenant = 'UDS_LOCAL_TEST' # 'uds_local_test' # 'uds_sandbox' + self.tenant_venue = 'DEV' # 'DEV1' # 'dev' + self.collection_name = 'KKK-09' # 'uds_collection' # 'sbx_collection' + self.collection_version = '24.03.20.14.40'.replace('.', '') # '2402011200' + self.collection_version = '001' + return + + def test_get_single_granule_01(self): + os.environ['ES_URL'] = 'vpc-uds-sbx-cumulus-es-qk73x5h47jwmela5nbwjte4yzq.us-west-2.es.amazonaws.com' + os.environ['ES_PORT'] = '9200' + collection_id = f'URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:{self.collection_name}___{self.collection_version}' + granule_id = f'{collection_id}:test_file09' + + mock_feature = { + 'archive_status': 'cnm_r_failed', + 'archive_error_message': 'testing 1 2 3', + 'archive_error_code': 'VALIDATION_ERROR', + + "type": "Feature", + "stac_version": "1.0.0", + "id": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001:test_file09", + "properties": { + "datetime": "2024-11-26T23:37:15.288000Z", + "start_datetime": "2016-01-31T18:00:00.009000Z", + "end_datetime": "2016-01-31T19:59:59.991000Z", + "created": "1970-01-01T00:00:00Z", + "updated": "2024-11-26T23:38:01.692000Z", + "status": "completed", + "provider": "unity", + }, + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.0 + ] + }, + "links": [ + { + "rel": "collection", + "href": "." + } + ], + "assets": { + "test_file09.nc": { + "href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001:test_file09/test_file09.nc", + "title": "test_file09.nc", + "description": "size=0;checksumType=md5;checksum=00000000000000000000000000000000;", + "file:size": 0, + "file:checksum": "00000000000000000000000000000000", + "roles": [ + "data" + ] + }, + "test_file09.nc.cas": { + "href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001:test_file09/test_file09.nc.cas", + "title": "test_file09.nc.cas", + "description": "size=0;checksumType=md5;checksum=00000000000000000000000000000000;", + "file:size": 0, + "file:checksum": "00000000000000000000000000000000", + "roles": [ + "metadata" + ] + }, + "test_file09.nc.stac.json": { + "href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001:test_file09/test_file09.nc.stac.json", + "title": "test_file09.nc.stac.json", + "description": "size=0;checksumType=md5;checksum=00000000000000000000000000000000;", + "file:size": 0, + "file:checksum": "00000000000000000000000000000000", + "roles": [ + "metadata" + ] + }, + "test_file09.cmr.xml": { + "href": "s3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001/URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001:test_file09/test_file09.cmr.xml", + "title": "test_file09.cmr.xml", + "description": "size=1716;checksumType=md5;checksum=f842ba4e23e76ae81014a01c820b01f7;", + "file:size": 1716, + "file:checksum": "f842ba4e23e76ae81014a01c820b01f7", + "roles": [ + "metadata" + ] + } + }, + "bbox": { + "type": "envelope", + "coordinates": [ + [ + -180.0, + 90.0 + ], + [ + 180.0, + -90.0 + ] + ] + }, + "stac_extensions": [ + "https://stac-extensions.github.io/file/v2.1.0/schema.json" + ], + "collection": "URN:NASA:UNITY:UDS_LOCAL_TEST:DEV:KKK-09___001", + "event_time": 1732664287722 + } + + granules_db_index = GranulesDbIndex() + granules_db_index.add_entry(self.tenant, self.tenant_venue, mock_feature, granule_id) + granules_dapa_query = GranulesDapaQueryEs(collection_id, 1, None, None, None, None, f'localhost/api-prefix') + granules_result = granules_dapa_query.get_single_granule(granule_id) + print(granules_result) + sample = { + 'type': 'Feature', + 'stac_version': '1.0.0', + 'id': 'URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53', + 'properties': {'tag': '#sample', 'c_data1': [1, 10, 100, 1000], 'c_data2': [False, True, True, False, True], 'c_data3': ['Bellman Ford'], 'soil10': {'0_0': 0, '0_1': 1, '0_2': 0}, 'datetime': '2024-10-01T13:12:11.810000Z', 'start_datetime': '2016-01-31T18:00:00.009000Z', 'end_datetime': '2016-01-31T19:59:59.991000Z', 'created': '1970-01-01T00:00:00Z', 'updated': '2024-10-01T13:12:55.423000Z', 'status': 'completed', 'provider': 'unity', 'archive_status': 'cnm_r_failed', 'archive_error_message': '[{"uri": "https://uds-distribution-placeholder/uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53/abcd.1234.efgh.test_file-24.08.13.13.53.data.stac.json", "error": "mismatched size: 11 v. -1"}]', 'archive_error_code': 'VALIDATION_ERROR'}, + 'geometry': {'type': 'Point', 'coordinates': [0.0, 0.0]}, 'links': [{'rel': 'collection', 'href': '.'}, {'rel': 'self', 'href': '/Users/wphyo/Projects/unity/unity-data-services/tests/cumulus_lambda_functions/uds_api/dapa/localhost/api-prefix/collections/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208/items/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53', 'type': 'application/json', 'title': 'URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53'}], 'assets': {'abcd.1234.efgh.test_file-24.08.13.13.53.data.stac.json': {'href': 's3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53/abcd.1234.efgh.test_file-24.08.13.13.53.data.stac.json', 'title': 'abcd.1234.efgh.test_file-24.08.13.13.53.data.stac.json', 'description': 'size=-1;checksumType=md5;checksum=unknown;', 'file:size': -1, 'file:checksum': 'unknown', 'roles': ['data']}, 'abcd.1234.efgh.test_file-24.08.13.13.53.cmr.xml': {'href': 's3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53/abcd.1234.efgh.test_file-24.08.13.13.53.cmr.xml', 'title': 'abcd.1234.efgh.test_file-24.08.13.13.53.cmr.xml', 'description': 'size=1812;checksumType=md5;checksum=38c9d99e56312b595faa5e99df30b175;', 'file:size': 1812, 'file:checksum': '38c9d99e56312b595faa5e99df30b175', 'roles': ['metadata']}, 'abcd.1234.efgh.test_file-24.08.13.13.53.nc.stac.json': {'href': 's3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53/abcd.1234.efgh.test_file-24.08.13.13.53.nc.stac.json', 'title': 'abcd.1234.efgh.test_file-24.08.13.13.53.nc.stac.json', 'description': 'size=-1;checksumType=md5;checksum=unknown;', 'file:size': -1, 'file:checksum': 'unknown', 'roles': ['metadata']}, 'abcd.1234.efgh.test_file-24.08.13.13.53.nc.cas': {'href': 's3://uds-sbx-cumulus-staging/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208:abcd.1234.efgh.test_file-24.08.13.13.53/abcd.1234.efgh.test_file-24.08.13.13.53.nc.cas', 'title': 'abcd.1234.efgh.test_file-24.08.13.13.53.nc.cas', 'description': 'size=-1;checksumType=md5;checksum=unknown;', 'file:size': -1, 'file:checksum': 'unknown', 'roles': ['metadata']}}, 'bbox': [-180.0, -90.0, 180.0, 90.0], 'stac_extensions': ['https://stac-extensions.github.io/file/v2.1.0/schema.json'], 'collection': 'URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2409301208'} + self.assertTrue('properties' in granules_result, 'missing properties in granule') + self.assertTrue('archive_status' in granules_result['properties'], 'missing archive_status in granule>properties') + self.assertTrue('archive_error_message' in granules_result['properties'], 'missing archive_error_message in granule>properties') + self.assertTrue('archive_error_code' in granules_result['properties'], 'missing archive_error_code in granule>properties') + self.assertFalse('archive_status' in granules_result, 'missing archive_status in granule') + self.assertFalse('archive_error_message' in granules_result, 'missing archive_error_message in granule') + self.assertFalse('archive_error_code' in granules_result, 'missing archive_error_code in granule') + + return diff --git a/tests/integration_tests/test_uds_api.py b/tests/integration_tests/test_uds_api.py index 0580e5a3..4869162a 100644 --- a/tests/integration_tests/test_uds_api.py +++ b/tests/integration_tests/test_uds_api.py @@ -184,7 +184,7 @@ def test_collections_get_single_granule(self): return def test_granules_get(self): - post_url = f'{self.uds_url}collections/URN:NASA:UNITY:UDS_MY_LOCAL_ARCHIVE_TEST:DEV:UDS_UNIT_COLLECTION___2408290522/items/' # MCP Dev + post_url = f'{self.uds_url}collections/urn:nasa:unity:unity:dev:SBG-L2A_RFL___1/items/' # MCP Dev headers = { 'Authorization': f'Bearer {self.bearer_token}', } From bf0dbd0c7c15b249cea0e7ad8b2a7b8a1a1d2ecf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 05:22:04 -0800 Subject: [PATCH 04/10] chore: update version + change log (#477) Co-authored-by: wphyojpl --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 022d15de..598ff719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.2] - 2024-12-09 +### Fixed +- [#448](https://github.com/unity-sds/unity-data-services/pull/448) fix: wrong location archive keys + ## [9.1.1] - 2024-12-09 ### Fixed - [#475](https://github.com/unity-sds/unity-data-services/pull/475) fix: index to es is not setting bbox correctly diff --git a/setup.py b/setup.py index e092e45a..e61258fd 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="cumulus_lambda_functions", - version="9.1.1", + version="9.1.2", packages=find_packages(), install_requires=install_requires, package_data={ From f38f817b0303e38b50ce3e69a7deb1eb542fd783 Mon Sep 17 00:00:00 2001 From: wphyojpl <38299756+wphyojpl@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:44:14 -0800 Subject: [PATCH 05/10] feat: Uds lib update (#478) * breaking: using latest uds-lib + update docker * feat: use latest uds-lib * feat: update dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 27a176c9..42e7ec78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ jsonschema==4.23.0 jsonschema-specifications==2023.12.1 lark==0.12.0 mangum==0.18.0 -mdps-ds-lib==1.1.0 +mdps-ds-lib==1.1.1 pydantic==2.9.2 pydantic_core==2.23.4 pygeofilter==0.2.4 From 58a697d3563a96556b844a01480b8d1ff77ec12d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 09:46:41 -0800 Subject: [PATCH 06/10] chore: update version + change log (#479) Co-authored-by: wphyojpl --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 598ff719..fcb549fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.2.0] - 2024-12-09 +### Changed +- [#478](https://github.com/unity-sds/unity-data-services/pull/478) feat: uds lib update + ## [9.1.2] - 2024-12-09 ### Fixed - [#448](https://github.com/unity-sds/unity-data-services/pull/448) fix: wrong location archive keys diff --git a/setup.py b/setup.py index e61258fd..c2437d7f 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="cumulus_lambda_functions", - version="9.1.2", + version="9.2.0", packages=find_packages(), install_requires=install_requires, package_data={ From d1da4c6da72e2e58372b8fcb45473ba59283a559 Mon Sep 17 00:00:00 2001 From: Galen Hollins <47542238+galenatjpl@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:11:43 -0800 Subject: [PATCH 07/10] updated name spaces and naming of resources (#482) --- tf-module/marketplace/outputs.tf | 4 +-- tf-module/marketplace/s3_bucket_creation.tf | 32 +++++++++---------- .../marketplace/terraform.tfvars.example | 4 +-- tf-module/marketplace/variables.tf | 8 ++--- tf-module/unity-cumulus/variables.tf | 4 +-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tf-module/marketplace/outputs.tf b/tf-module/marketplace/outputs.tf index 9c7661e8..c1fe2db9 100644 --- a/tf-module/marketplace/outputs.tf +++ b/tf-module/marketplace/outputs.tf @@ -1,3 +1,3 @@ -output "market_bucket_name" { - value = aws_s3_bucket.market_bucket.id +output "datastore_bucket_name" { + value = aws_s3_bucket.datastore_bucket.id } \ No newline at end of file diff --git a/tf-module/marketplace/s3_bucket_creation.tf b/tf-module/marketplace/s3_bucket_creation.tf index 91307af8..0664f028 100644 --- a/tf-module/marketplace/s3_bucket_creation.tf +++ b/tf-module/marketplace/s3_bucket_creation.tf @@ -7,11 +7,11 @@ locals { "Env" = var.venue "ServiceArea" = "ds" "CapVersion" = "1.0.0" - "Component" = "MarketplaceBucket" + "Component" = "DatastoreBucket" "CreatedBy" = "ds" - "Stack" = "MarketplaceBucket" - "Capability" = "data-storage" - "Name" = "${var.project}-${var.venue}-ds-data-storage-marketplace-bucket" + "Stack" = "DatastoreBucket" + "Capability" = "datastore" + "Name" = "${var.project}-${var.venue}-ds-datastore-bucket" } ) } @@ -27,13 +27,13 @@ data "aws_ssm_parameter" "uds_aws_account_region" { data "aws_ssm_parameter" "uds_prefix" { name = "arn:aws:ssm:${data.aws_ssm_parameter.uds_aws_account_region.value}:${data.aws_ssm_parameter.uds_aws_account.value}:parameter${var.uds_prefix_ssm_path}" } -resource "aws_s3_bucket" "market_bucket" { - bucket = lower(replace("${var.project}-${var.venue}-unity-${var.market_bucket_name}", "_", "-")) +resource "aws_s3_bucket" "datastore_bucket" { + bucket = lower(replace("${var.project}-${var.venue}-unity-${var.datastore_bucket_name}", "_", "-")) tags = local.bucket_tags } -resource "aws_s3_bucket_server_side_encryption_configuration" "market_bucket" { // https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration - bucket = aws_s3_bucket.market_bucket.id +resource "aws_s3_bucket_server_side_encryption_configuration" "datastore_bucket" { // https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration + bucket = aws_s3_bucket.datastore_bucket.id rule { apply_server_side_encryption_by_default { sse_algorithm = "AES256" @@ -41,30 +41,30 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "market_bucket" { } } -resource "aws_s3_bucket_policy" "market_bucket" { +resource "aws_s3_bucket_policy" "datastore_bucket" { // https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_policy - bucket = aws_s3_bucket.market_bucket.id + bucket = aws_s3_bucket.datastore_bucket.id policy = templatefile("${path.module}/s3_bucket_policy.json", { udsAwsAccount: data.aws_ssm_parameter.uds_aws_account.value, - s3BucketName: aws_s3_bucket.market_bucket.id, + s3BucketName: aws_s3_bucket.datastore_bucket.id, cumulus_lambda_processing_role_name: "${data.aws_ssm_parameter.uds_prefix.value}-${var.cumulus_lambda_processing_role_name_postfix}", cumulus_sf_lambda_role_name: "${data.aws_ssm_parameter.uds_prefix.value}${var.cumulus_sf_lambda_role_name_postfix}", }) } resource "aws_s3_bucket_notification" "bucket_notification" { - bucket = aws_s3_bucket.market_bucket.id + bucket = aws_s3_bucket.datastore_bucket.id topic { topic_arn = "arn:aws:sns:${data.aws_ssm_parameter.uds_aws_account_region.value}:${data.aws_ssm_parameter.uds_aws_account.value}:${data.aws_ssm_parameter.uds_prefix.value}-granules_cnm_ingester" events = ["s3:ObjectCreated:*"] filter_suffix = ".json" - filter_prefix = var.market_bucket__notification_prefix + filter_prefix = var.datastore_bucket_notification_prefix } } -resource "aws_ssm_parameter" "primary_data_bucket" { - name = "/unity/ds/data/bucket/primary-data-bucket" +resource "aws_ssm_parameter" "datastore_bucket" { + name = "/unity/${var.project}/${var.venue}/ds/datastore-bucket" type = "String" - value = aws_s3_bucket.market_bucket.bucket + value = aws_s3_bucket.datastore_bucket.bucket tags = local.bucket_tags } \ No newline at end of file diff --git a/tf-module/marketplace/terraform.tfvars.example b/tf-module/marketplace/terraform.tfvars.example index 1df5bea3..aa150cbc 100644 --- a/tf-module/marketplace/terraform.tfvars.example +++ b/tf-module/marketplace/terraform.tfvars.example @@ -1,7 +1,7 @@ project = "Unity" venue = "dev" -market_bucket_name = "william_test_1" -market_bucket__notification_prefix = "stage_out" +datastore_bucket_name = "william_test_1" +datastore_bucket_notification_prefix = "stage_out" uds_aws_account_ssm_path="/unity/shared-services/aws/account" uds_aws_account_region_ssm_path="/unity/shared-services/aws/account/region" uds_prefix_ssm_path="/unity/shared-services/data-catalog/deployment/prefix" \ No newline at end of file diff --git a/tf-module/marketplace/variables.tf b/tf-module/marketplace/variables.tf index fc6218c4..d2594963 100644 --- a/tf-module/marketplace/variables.tf +++ b/tf-module/marketplace/variables.tf @@ -1,12 +1,12 @@ variable "installprefix" { type = string default = "" - description = "This is not needed, but required by UCS marketplace. Empty string is good enough for manual deployment" + description = "This is not needed, but required by UCS Marketplace. Empty string is good enough for manual deployment" } variable "deployment_name" { type = string default = "" - description = "This is not needed, but required by UCS marketplace. Empty string is good enough for manual deployment" + description = "This is not needed, but required by UCS Marketplace. Empty string is good enough for manual deployment" } variable "project" { type = string @@ -23,11 +23,11 @@ variable "tags" { type = map(string) default = {} } -variable "market_bucket_name" { +variable "datastore_bucket_name" { type = string description = "name of S3 bucket. Note-1: it will be prefixed with '--unity-'. Note-2: It should only have '-'. '_' will be replaced with '-'" } -variable "market_bucket__notification_prefix" { +variable "datastore_bucket_notification_prefix" { type = string default = "stage_out" description = "path to the directory where catalogs.json will be written" diff --git a/tf-module/unity-cumulus/variables.tf b/tf-module/unity-cumulus/variables.tf index ee5e2bf1..04508a25 100644 --- a/tf-module/unity-cumulus/variables.tf +++ b/tf-module/unity-cumulus/variables.tf @@ -176,13 +176,13 @@ variable "cors_integration_response" { variable "health_check_marketplace_item" { type = string default = "shared-services" - description = "name of the portion of market place item as path of SSM token" + description = "name of the portion of Marketplace item as path of SSM token" } variable "health_check_component_name" { type = string default = "data-catalog" - description = "name of the portion of market place item as path of SSM token" + description = "name of the portion of Marketplace item as path of SSM token" } variable "is_deploying_healthcheck" { From d9ded155f2cdcf39531bd07b2f9ab56d0a57a733 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:52:48 -0500 Subject: [PATCH 08/10] chore: update version + change log (#483) Co-authored-by: wphyojpl --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb549fa..7310bdb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.3.0] - 2024-12-11 +### Changed +- [#482](https://github.com/unity-sds/unity-data-services/pull/482) feat: updated name spaces and naming of resources + ## [9.2.0] - 2024-12-09 ### Changed - [#478](https://github.com/unity-sds/unity-data-services/pull/478) feat: uds lib update diff --git a/setup.py b/setup.py index c2437d7f..ae5f279f 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="cumulus_lambda_functions", - version="9.2.0", + version="9.3.0", packages=find_packages(), install_requires=install_requires, package_data={ From a755b6c4998cbbe8c5f82461e77f8648da768b78 Mon Sep 17 00:00:00 2001 From: wphyojpl <38299756+wphyojpl@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:37:01 -0800 Subject: [PATCH 09/10] feat: Delete granule endpoint (#485) * breaking: using latest uds-lib + update docker * feat: use latest uds-lib * feat: add delete single granule endpoint * fix: deleting individually from real index. not from alias * feat: add test case * fix: add optional delete files flag * fix: wrong authorization action * fix: temp disable cumulus delete * fix: disable s3 deletion to see if cumulus delete it * fix: update test --- .../cumulus_wrapper/query_granules.py | 44 +++++++++ .../lib/uds_db/granules_db_index.py | 17 ++++ .../uds_api/granules_api.py | 50 ++++++++++ requirements.txt | 2 +- .../test_granules_deletion.py | 95 +++++++++++++++++++ .../test_stage_out_ingestion.py | 4 +- 6 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 tests/integration_tests/test_granules_deletion.py diff --git a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py index cc4a4829..86eacefb 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/query_granules.py +++ b/cumulus_lambda_functions/cumulus_wrapper/query_granules.py @@ -21,6 +21,7 @@ def __init__(self, cumulus_base: str, cumulus_token: str): super().__init__(cumulus_base, cumulus_token) self._conditions.append('status=completed') self._item_transformer = ItemTransformer() + self.__collection_id = None def with_filter(self, filter_key, filter_values: list): if len(filter_values) < 1: @@ -34,6 +35,7 @@ def with_filter(self, filter_key, filter_values: list): def with_collection_id(self, collection_id: str): self._conditions.append(f'{self.__collection_id_key}={collection_id}') + self.__collection_id = collection_id return self def with_bbox(self): @@ -130,6 +132,48 @@ def query_direct_to_private_api(self, private_api_prefix: str, transform=True): return {'server_error': f'error while invoking:{str(e)}'} return {'results': stac_list} + def delete_entry(self, private_api_prefix: str, granule_id: str): + payload = { + 'httpMethod': 'DELETE', + 'resource': '/{proxy+}', + 'path': f'/{self.__granules_key}/{self.__collection_id}/{granule_id}', + 'queryStringParameters': {**{k[0]: k[1] for k in [k1.split('=') for k1 in self._conditions]}}, + # 'queryStringParameters': {'limit': '30'}, + 'headers': { + 'Content-Type': 'application/json', + }, + # 'body': json.dumps({"action": "removeFromCmr"}) + } + LOGGER.debug(f'payload: {payload}') + try: + query_result = self._invoke_api(payload, private_api_prefix) + """ + {'statusCode': 200, 'body': '{"meta":{"name":"cumulus-api","stack":"am-uds-dev-cumulus","table":"granule","limit":3,"page":1,"count":0},"results":[]}', 'headers': {'x-powered-by': 'Express', 'access-control-allow-origin': '*', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'content-type': 'application/json; charset=utf-8', 'content-length': '120', 'etag': 'W/"78-YdHqDNIH4LuOJMR39jGNA/23yOQ"', 'date': 'Tue, 07 Jun 2022 22:30:44 GMT', 'connection': 'close'}, 'isBase64Encoded': False} + """ + if query_result['statusCode'] >= 500: + LOGGER.error(f'server error status code: {query_result.statusCode}. details: {query_result}') + return {'server_error': query_result} + if query_result['statusCode'] >= 400: + LOGGER.error(f'client error status code: {query_result.statusCode}. details: {query_result}') + return {'client_error': query_result} + query_result = json.loads(query_result['body']) + LOGGER.info(f'json query_result: {query_result}') + """ + { + "detail": "Record deleted" + } + """ + if 'detail' not in query_result: + LOGGER.error(f'missing key: detail. invalid response json: {query_result}') + return {'server_error': f'missing key: detail. invalid response json: {query_result}'} + if query_result['detail'] != 'Record deleted': + LOGGER.error(f'Wrong Message: {query_result}') + return {'server_error': f'Wrong Message: {query_result}'} + except Exception as e: + LOGGER.exception('error while invoking') + return {'server_error': f'error while invoking:{str(e)}'} + return {} + def query(self, transform=True): conditions_str = '&'.join(self._conditions) LOGGER.info(f'cumulus_base: {self.cumulus_base}') diff --git a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py index f993f16e..24139081 100644 --- a/cumulus_lambda_functions/lib/uds_db/granules_db_index.py +++ b/cumulus_lambda_functions/lib/uds_db/granules_db_index.py @@ -201,6 +201,23 @@ def get_entry(self, tenant: str, tenant_venue: str, doc_id: str, ): raise ValueError(f"no such granule: {doc_id}") return result + def delete_entry(self, tenant: str, tenant_venue: str, doc_id: str, ): + read_alias_name = f'{DBConstants.granules_read_alias_prefix}_{tenant}_{tenant_venue}'.lower().strip() + result = self.__es.query({ + 'size': 9999, + 'query': {'term': {'_id': doc_id}} + }, read_alias_name) + if result is None: + raise ValueError(f"no such granule: {doc_id}") + for each_granule in result['hits']['hits']: + delete_result = self.__es.delete_by_query({ + 'query': {'term': {'_id': each_granule['_id']}} + }, each_granule['_index']) + LOGGER.debug(f'delete_result: {delete_result}') + if delete_result is None: + raise ValueError(f"error deleting {each_granule}") + return result + def update_entry(self, tenant: str, tenant_venue: str, json_body: dict, doc_id: str, ): write_alias_name = f'{DBConstants.granules_write_alias_prefix}_{tenant}_{tenant_venue}'.lower().strip() json_body['event_time'] = TimeUtils.get_current_unix_milli() diff --git a/cumulus_lambda_functions/uds_api/granules_api.py b/cumulus_lambda_functions/uds_api/granules_api.py index a3961d08..86e8dc23 100644 --- a/cumulus_lambda_functions/uds_api/granules_api.py +++ b/cumulus_lambda_functions/uds_api/granules_api.py @@ -2,6 +2,11 @@ import os from typing import Union +from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 +from pystac import Item + +from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery + from cumulus_lambda_functions.daac_archiver.daac_archiver_logic import DaacArchiverLogic from cumulus_lambda_functions.uds_api.dapa.daac_archive_crud import DaacArchiveCrud, DaacDeleteModel, DaacAddModel, \ DaacUpdateModel @@ -239,6 +244,51 @@ async def get_single_granule_dapa(request: Request, collection_id: str, granule_ raise HTTPException(status_code=500, detail=str(e)) return granules_result +@router.delete("/{collection_id}/items/{granule_id}") +@router.delete("/{collection_id}/items/{granule_id}/") +async def delete_single_granule_dapa(request: Request, collection_id: str, granule_id: str): + authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory() \ + .get_instance(UDSAuthorizerFactory.cognito, + es_url=os.getenv('ES_URL'), + es_port=int(os.getenv('ES_PORT', '443')) + ) + auth_info = FastApiUtils.get_authorization_info(request) + collection_identifier = UdsCollections.decode_identifier(collection_id) + if not authorizer.is_authorized_for_collection(DBConstants.delete, collection_id, + auth_info['ldap_groups'], + collection_identifier.tenant, + collection_identifier.venue): + LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}') + raise HTTPException(status_code=403, detail=json.dumps({ + 'message': 'not authorized to execute this action' + })) + try: + LOGGER.debug(f'deleting granule: {granule_id}') + cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX') + cumulus = GranulesQuery('https://na/dev', 'NA') + cumulus.with_collection_id(collection_id) + cumulus_delete_result = cumulus.delete_entry(cumulus_lambda_prefix, granule_id) # TODO not sure it is correct granule ID + LOGGER.debug(f'cumulus_delete_result: {cumulus_delete_result}') + es_delete_result = GranulesDbIndex().delete_entry(collection_identifier.tenant, + collection_identifier.venue, + granule_id + ) + LOGGER.debug(f'es_delete_result: {es_delete_result}') + # es_delete_result = [Item.from_dict(k['_source']) for k in es_delete_result['hits']['hits']] + # if delete_files is False: + # LOGGER.debug(f'Not deleting files as it is set to false in the request') + # return {} + # s3 = AwsS3() + # for each_granule in es_delete_result: + # s3_urls = [v.href for k, v in each_granule.assets.items()] + # LOGGER.debug(f'deleting S3 for {each_granule.id} - s3_urls: {s3_urls}') + # delete_result = s3.delete_multiple(s3_urls=s3_urls) + # LOGGER.debug(f'delete_result for {each_granule.id} - delete_result: {delete_result}') + except Exception as e: + LOGGER.exception('failed during get_granules_dapa') + raise HTTPException(status_code=500, detail=str(e)) + return {} + @router.put("/{collection_id}/archive/{granule_id}") @router.put("/{collection_id}/archive/{granule_id}/") diff --git a/requirements.txt b/requirements.txt index 42e7ec78..72949417 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ jsonschema==4.23.0 jsonschema-specifications==2023.12.1 lark==0.12.0 mangum==0.18.0 -mdps-ds-lib==1.1.1 +mdps-ds-lib==1.1.1.dev000200 pydantic==2.9.2 pydantic_core==2.23.4 pygeofilter==0.2.4 diff --git a/tests/integration_tests/test_granules_deletion.py b/tests/integration_tests/test_granules_deletion.py new file mode 100644 index 00000000..b05acd45 --- /dev/null +++ b/tests/integration_tests/test_granules_deletion.py @@ -0,0 +1,95 @@ +import base64 +import json +import os +from unittest import TestCase + +import requests +from dotenv import load_dotenv +from mdps_ds_lib.lib.aws.aws_s3 import AwsS3 +from mdps_ds_lib.lib.cognito_login.cognito_login import CognitoLogin + + +class TestGranulesDeletion(TestCase): + def setUp(self) -> None: + super().setUp() + load_dotenv() + self._url_prefix = f'{os.environ.get("UNITY_URL")}/{os.environ.get("UNITY_STAGE", "sbx-uds-dapa")}' + self.cognito_login = CognitoLogin() \ + .with_client_id(os.environ.get('CLIENT_ID', '')) \ + .with_cognito_url(os.environ.get('COGNITO_URL', '')) \ + .with_verify_ssl(False) \ + .start(base64.standard_b64decode(os.environ.get('USERNAME')).decode(), + base64.standard_b64decode(os.environ.get('PASSWORD')).decode()) + self.bearer_token = self.cognito_login.token + self.stage = os.environ.get("UNITY_URL").split('/')[-1] + self.uds_url = f'{os.environ.get("UNITY_URL")}/{os.environ.get("UNITY_STAGE", "sbx-uds-dapa")}/' + self.custom_metadata_body = { + 'tag': {'type': 'keyword'}, + 'c_data1': {'type': 'long'}, + 'c_data2': {'type': 'boolean'}, + 'c_data3': {'type': 'keyword'}, + } + + self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' + self.tenant_venue = 'DEV' # 'DEV1' # 'dev' + self.collection_name = 'CCC-04' # 'uds_collection' # 'sbx_collection' + self.collection_version = '08'.replace('.', '') # '2402011200' + return + + def test_01_setup_permissions(self): + collection_url = f'{self._url_prefix}/admin/auth' + admin_add_body = { + "actions": ["READ", "CREATE", "DELETE"], + "resources": [f"URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:.*"], + "tenant": self.tenant, + "venue": self.tenant_venue, + "group_name": "Unity_Viewer" + } + s = requests.session() + s.trust_env = False + response = s.put(url=collection_url, headers={ + 'Authorization': f'Bearer {self.cognito_login.token}', + 'Content-Type': 'application/json', + }, verify=False, data=json.dumps(admin_add_body)) + self.assertEqual(response.status_code, 200, f'wrong status code: {response.text}') + response_json = response.content.decode() + print(response_json) + return + + def test_delete_all(self): + collection_id = f'URN:NASA:UNITY:{self.tenant}:{self.tenant_venue}:{self.collection_name}___001' + post_url = f'{self.uds_url}collections/{collection_id}/items/' # MCP Dev + headers = { + 'Authorization': f'Bearer {self.bearer_token}', + } + print(post_url) + query_result = requests.get(url=post_url, + headers=headers, + ) + self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') + response_json = json.loads(query_result.text) + print(json.dumps(response_json, indent=4)) + self.assertTrue(len(response_json['features']) > 0, f'empty collection :(') + deleting_granule_id = response_json['features'][0]['id'] + + asset_urls = [v['href'] for k, v in response_json['features'][0]['assets'].items()] + print(asset_urls) + post_url = f'{self.uds_url}collections/{collection_id}/items/{deleting_granule_id}/' # MCP Dev + print(post_url) + query_result = requests.delete(url=post_url, + headers=headers, + ) + self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') + response_json = json.loads(query_result.text) + print(json.dumps(response_json, indent=4)) + + post_url = f'{self.uds_url}collections/{collection_id}/items/' # MCP Dev + query_result = requests.get(url=post_url, headers=headers,) + self.assertEqual(query_result.status_code, 200, f'wrong status code. {query_result.text}') + response_json = json.loads(query_result.text) + print(json.dumps(response_json, indent=4)) + + s3 = AwsS3() + for each_url in asset_urls: + self.assertFalse(s3.set_s3_url(each_url).exists(s3.target_bucket, s3.target_key), f'file still exists: {each_url}') + return diff --git a/tests/integration_tests/test_stage_out_ingestion.py b/tests/integration_tests/test_stage_out_ingestion.py index ca4ba83d..f72b6e77 100644 --- a/tests/integration_tests/test_stage_out_ingestion.py +++ b/tests/integration_tests/test_stage_out_ingestion.py @@ -42,7 +42,7 @@ def setUp(self) -> None: self.tenant = 'UDS_LOCAL_TEST_3' # 'uds_local_test' # 'uds_sandbox' self.tenant_venue = 'DEV' # 'DEV1' # 'dev' - self.collection_name = 'AAA' # 'uds_collection' # 'sbx_collection' + self.collection_name = 'CCC' # 'uds_collection' # 'sbx_collection' self.collection_version = '24.03.20.14.40'.replace('.', '') # '2402011200' return @@ -232,7 +232,7 @@ def test_03_upload_complete_catalog_role_as_key(self): "type": "Point", "coordinates": [0.0, 0.0] }, - bbox=[0.0, 0.1, 0.1, 0.0], + bbox=[0.0, 0.0, 0.1, 0.1], datetime=TimeUtils().parse_from_unix(0, True).get_datetime_obj(), properties={ "start_datetime": "2016-01-31T18:00:00.009057Z", From 75f029dd1dba32a0292929288ba248ec02192b14 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:41:56 -0800 Subject: [PATCH 10/10] chore: update version + change log (#487) Co-authored-by: ngachung --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7310bdb0..cfddfd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.4.0] - 2024-12-16 +### Changed +- [#485](https://github.com/unity-sds/unity-data-services/pull/485) feat: delete granule endpoint + ## [9.3.0] - 2024-12-11 ### Changed - [#482](https://github.com/unity-sds/unity-data-services/pull/482) feat: updated name spaces and naming of resources diff --git a/setup.py b/setup.py index ae5f279f..9604478e 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="cumulus_lambda_functions", - version="9.3.0", + version="9.4.0", packages=find_packages(), install_requires=install_requires, package_data={