Skip to content

Commit fe5d89d

Browse files
ngachungngachung
authored
Merge pull request #78 from unity-sds/develop
release/1.7.0
2 parents f969a95 + dd4e8d4 commit fe5d89d

File tree

28 files changed

+891
-117
lines changed

28 files changed

+891
-117
lines changed

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@ scratch*
55
local*
66
*egg-info*
77
dist
8-
__pycache__
8+
__pycache__
9+
.env

‎CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [1.7.0] - 2022-09-06
9+
### Added
10+
- [#62](https://github.com/unity-sds/unity-data-services/issues/66) Added OpenAPI spec for DAPA endpoints
11+
- [#66](https://github.com/unity-sds/unity-data-services/issues/66) Added pagination links to STAC response of DAPA endpoints
12+
- [#64](https://github.com/unity-sds/unity-data-services/issues/64) Added temporal coverage to DAPA collection endpoint
13+
### Changed
14+
- [#67](https://github.com/unity-sds/unity-data-services/issues/67) Updated STAC collection schema to be compatible with PySTAC library
15+
### Fixed
16+
817
## [1.6.17] - 2022-07-28
918
### Added
1019
### Fixed

‎ci.cd/Makefile

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,24 @@ export VERSION ?= latest
44

55

66
all: build_lambda upload_lambda update_lambda_function build_docker
7-
7+
local: build_lambda upload_lambda update_lambda_function_1 update_lambda_function_2 update_lambda_function_3
88
build_docker:
99
docker build -t "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" -f docker/Dockerfile .
1010

1111
zip_docker:
1212
docker save "$(IMAGE_PREFIX)/$(NAME):$(VERSION)" | gzip > "$(NAME)__$(VERSION).tar.gz"
1313

1414
build_lambda:
15-
docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.7 ci.cd/create_s3_zip.sh
15+
docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" cae-artifactory.jpl.nasa.gov:17001/python:3.9 ci.cd/create_s3_zip.sh
1616

1717
build_lambda_public:
1818
docker run --rm -v `PWD`:"/usr/src/app/cumulus_lambda_functions":z -w "/usr/src/app/cumulus_lambda_functions" python:3.7 ci.cd/create_s3_zip.sh
1919

2020
upload_lambda:
2121
aws --profile saml-pub s3 cp cumulus_lambda_functions_deployment.zip s3://am-uds-dev-cumulus-tf-state/unity_cumulus_lambda/
22-
23-
update_lambda_function:
24-
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:Test1 --publish
22+
update_lambda_function_1:
23+
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_dapa --publish &>/dev/null
24+
update_lambda_function_2:
25+
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_granules_dapa --publish &>/dev/null
26+
update_lambda_function_3:
27+
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_ingest_cnm_dapa --publish &>/dev/null

‎ci.cd/create_s3_zip.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ zip_file="${project_root_dir}/$ZIP_NAME" ; # save the result file in current wor
88

99
tmp_proj='/tmp/cumulus_lambda_functions'
1010

11-
source_dir="/usr/local/lib/python3.7/site-packages/"
11+
source_dir="/usr/local/lib/python3.9/site-packages/"
1212

1313
mkdir -p "$tmp_proj/cumulus_lambda_functions" && \
1414
cd $tmp_proj && \

‎cognito_readme.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@
99
"ClientId" : "7a1fglm2d54eoggj13lccivp25"
1010
}
1111
- ask U-CS to create credentials and change password the first time
12-
- run this command:
12+
- run this command (JPL AWS):
1313

1414
curl -X POST --data @cognito.jpl.aws.json -H 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' -H 'Content-Type: application/x-amz-json-1.1' https://cognito-idp.us-west-2.amazonaws.com/|jq
15+
curl -X POST --data @cognito.mcp.test.aws.json -H 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' -H 'Content-Type: application/x-amz-json-1.1' https://cognito-idp.us-west-2.amazonaws.com/|jq
16+
curl -X POST --data @cognito.mcp.dev.aws.json -H 'X-Amz-Target: AWSCognitoIdentityProviderService.InitiateAuth' -H 'Content-Type: application/x-amz-json-1.1' https://cognito-idp.us-west-2.amazonaws.com/|jq
1517
- successful response:
1618

1719
{

‎cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery
55
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
6+
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils
67

78
LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
89

@@ -37,25 +38,50 @@ def __assign_values(self):
3738
self.__offset = int(query_str_dict['offset'])
3839
return self
3940

41+
def __get_size(self):
42+
try:
43+
cumulus_size = self.__cumulus.get_size(self.__cumulus_lambda_prefix)
44+
except:
45+
LOGGER.exception(f'cannot get cumulus_size')
46+
cumulus_size = {'total_size': -1}
47+
return cumulus_size
48+
49+
def __get_pagination_urls(self):
50+
try:
51+
pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
52+
except Exception as e:
53+
LOGGER.exception(f'error while generating pagination links')
54+
return [{'message': f'error while generating pagination links: {str(e)}'}]
55+
return pagination_links
56+
4057
def start(self):
4158
try:
4259
cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
43-
except Exception as e:
60+
if 'server_error' in cumulus_result:
61+
return {
62+
'statusCode': 500,
63+
'body': {'message': cumulus_result['server_error']}
64+
}
65+
if 'client_error' in cumulus_result:
66+
return {
67+
'statusCode': 400,
68+
'body': {'message': cumulus_result['client_error']}
69+
}
70+
cumulus_size = self.__get_size()
4471
return {
45-
'statusCode': 500,
46-
'body': {'message': f'unpredicted error: {str(e)}'}
72+
'statusCode': 200,
73+
'body': json.dumps({
74+
'numberMatched': cumulus_size['total_size'],
75+
'numberReturned': len(cumulus_result['results']),
76+
'stac_version': '1.0.0',
77+
'type': 'FeatureCollection',
78+
'links': self.__get_pagination_urls(),
79+
'features': cumulus_result['results'],
80+
})
4781
}
48-
if 'server_error' in cumulus_result:
82+
except Exception as e:
83+
LOGGER.exception(f'unexpected error')
4984
return {
5085
'statusCode': 500,
51-
'body': {'message': cumulus_result['server_error']}
52-
}
53-
if 'client_error' in cumulus_result:
54-
return {
55-
'statusCode': 400,
56-
'body': {'message': cumulus_result['client_error']}
86+
'body': {'message': f'unpredicted error: {str(e)}'}
5787
}
58-
return {
59-
'statusCode': 200,
60-
'body': json.dumps({'features': cumulus_result['results']})
61-
}

‎cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery
55
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
6+
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils
67

78
LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
89

@@ -87,25 +88,50 @@ def __assign_values(self):
8788
self.__offset = int(query_str_dict['offset'])
8889
return self
8990

91+
def __get_size(self):
92+
try:
93+
cumulus_size = self.__cumulus.get_size(self.__cumulus_lambda_prefix)
94+
except:
95+
LOGGER.exception(f'cannot get cumulus_size')
96+
cumulus_size = {'total_size': -1}
97+
return cumulus_size
98+
99+
def __get_pagination_urls(self):
100+
try:
101+
pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
102+
except Exception as e:
103+
LOGGER.exception(f'error while generating pagination links')
104+
return [{'message': f'error while generating pagination links: {str(e)}'}]
105+
return pagination_links
106+
90107
def start(self):
91108
try:
92109
cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
93-
except Exception as e:
110+
if 'server_error' in cumulus_result:
111+
return {
112+
'statusCode': 500,
113+
'body': {'message': cumulus_result['server_error']}
114+
}
115+
if 'client_error' in cumulus_result:
116+
return {
117+
'statusCode': 400,
118+
'body': {'message': cumulus_result['client_error']}
119+
}
120+
cumulus_size = self.__get_size()
94121
return {
95-
'statusCode': 500,
96-
'body': {'message': f'unpredicted error: {str(e)}'}
122+
'statusCode': 200,
123+
'body': json.dumps({
124+
'numberMatched': cumulus_size['total_size'],
125+
'numberReturned': len(cumulus_result['results']),
126+
'stac_version': '1.0.0',
127+
'type': 'FeatureCollection', # TODO correct name?
128+
'links': self.__get_pagination_urls(),
129+
'features': cumulus_result['results']
130+
})
97131
}
98-
if 'server_error' in cumulus_result:
132+
except Exception as e:
133+
LOGGER.exception(f'unexpected error')
99134
return {
100135
'statusCode': 500,
101-
'body': {'message': cumulus_result['server_error']}
102-
}
103-
if 'client_error' in cumulus_result:
104-
return {
105-
'statusCode': 400,
106-
'body': {'message': cumulus_result['client_error']}
136+
'body': {'message': f'unpredicted error: {str(e)}'}
107137
}
108-
return {
109-
'statusCode': 200,
110-
'body': json.dumps({'features': cumulus_result['results']})
111-
}

‎cumulus_lambda_functions/cumulus_stac/collection_transformer.py

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
import json
2+
from datetime import datetime
3+
from urllib.parse import quote_plus
4+
25

36
from cumulus_lambda_functions.cumulus_stac.stac_transformer_abstract import StacTransformerAbstract
47

@@ -318,9 +321,12 @@ def __convert_to_stac_links(self, collection_file_obj: dict):
318321
href_link[0] = collection_file_obj['bucket']
319322
if 'regex' in collection_file_obj:
320323
href_link[1] = collection_file_obj['regex']
321-
stac_link['href'] = '___'.join(href_link)
324+
stac_link['href'] = f"./collection.json?bucket={href_link[0]}&regex={quote_plus(href_link[1])}"
322325
return stac_link
323326

327+
# def to_pystac_link_obj(self, input_dict: dict):
328+
# return
329+
324330
def to_stac(self, source: dict) -> dict:
325331
source_sample = {
326332
"createdAt": 1647992847582,
@@ -366,6 +372,26 @@ def to_stac(self, source: dict) -> dict:
366372
"url_path": "{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}",
367373
"timestamp": 1647992849273
368374
}
375+
# TemporalIntervals([
376+
# datetime.strptime(source['dateFrom'])
377+
# ])
378+
# stac_collection = pystac.Collection(
379+
# id=f"{source['name']}___{source['version']}",
380+
# description='TODO',
381+
# extent=Extent(
382+
# SpatialExtent([[0, 0, 0, 0]]),
383+
# TemporalExtent([[source['dateFrom'] if 'dateFrom' in source else None,
384+
# source['dateTo'] if 'dateTo' in source else None]])
385+
# ),
386+
# summaries=Summaries({
387+
# "granuleId": [source['granuleId'] if 'granuleId' in source else ''],
388+
# "granuleIdExtraction": [source['granuleIdExtraction'] if 'granuleIdExtraction' in source else ''],
389+
# "process": [source['process'] if 'process' in source else ''],
390+
# "totalGranules": [source['total_size'] if 'total_size' in source else -1],
391+
# }),
392+
# )
393+
# stac_collection.get_root_link().target = './collection.json'
394+
# stac_collection.add_links([Link.from_dict(k) for k in [self.__convert_to_stac_links(k) for k in source['files']]])
369395
stac_collection = {
370396
"type": "Collection",
371397
"stac_version": "1.0.0",
@@ -380,16 +406,24 @@ def to_stac(self, source: dict) -> dict:
380406
"bbox": [[0, 0, 0, 0]]
381407
},
382408
"temporal": {
383-
"interval": [[None, None]]
409+
"interval": [[source['dateFrom'] if 'dateFrom' in source else None,
410+
source['dateTo'] if 'dateTo' in source else None
411+
]]
384412
}
385413
},
386414
"assets": {},
387415
"summaries": {
388-
"granuleId": source['granuleId'] if 'granuleId' in source else '',
389-
"granuleIdExtraction": source['granuleIdExtraction'] if 'granuleIdExtraction' in source else '',
390-
"process": source['process'] if 'process' in source else '',
416+
"granuleId": [source['granuleId'] if 'granuleId' in source else ''],
417+
"granuleIdExtraction": [source['granuleIdExtraction'] if 'granuleIdExtraction' in source else ''],
418+
"process": [source['process'] if 'process' in source else ''],
419+
"totalGranules": [source['total_size'] if 'total_size' in source else -1],
391420
},
392-
"links": [self.__convert_to_stac_links(k) for k in source['files']],
421+
"links": [{
422+
"rel": "root",
423+
"type": "application/json",
424+
"title": f"{source['name']}___{source['version']}",
425+
"href": "./collection.json"
426+
}] + [self.__convert_to_stac_links(k) for k in source['files']],
393427
}
394428
return stac_collection
395429

‎cumulus_lambda_functions/cumulus_upload_granules/upload_granules.py

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
1-
import json
21
import logging
32
import os
43
import re
54
from collections import defaultdict
65
from glob import glob
7-
8-
import requests
6+
from urllib.parse import urlparse, unquote_plus
97

108
from cumulus_lambda_functions.cumulus_dapa_client.dapa_client import DapaClient
119
from cumulus_lambda_functions.lib.aws.aws_s3 import AwsS3
@@ -50,9 +48,19 @@ def __set_props_from_env(self):
5048
self.__delete_files = os.environ.get(self.DELETE_FILES_KEY, 'FALSE').strip().upper() == 'TRUE'
5149
return self
5250

51+
def __get_href(self, input_href: str):
52+
parse_result = urlparse(input_href)
53+
if parse_result.query == '':
54+
return ''
55+
query_dict = [k.split('=') for k in parse_result.query.split('&')]
56+
query_dict = {k[0]: unquote_plus(k[1]) for k in query_dict}
57+
if 'regex' not in query_dict:
58+
raise ValueError(f'missing regex in {input_href}')
59+
return query_dict['regex']
60+
5361
def __sort_granules(self):
54-
file_regex_list = {k['type']: k['href'].split('___')[-1] for k in self.__collection_details['links'] if not k['title'].endswith('cmr.xml')}
55-
granule_id_extraction = self.__collection_details['summaries']['granuleIdExtraction']
62+
file_regex_list = {k['type']: self.__get_href(k['href']) for k in self.__collection_details['links'] if k['rel'] != 'root' and not k['title'].endswith('cmr.xml')}
63+
granule_id_extraction = self.__collection_details['summaries']['granuleIdExtraction'][0]
5664
granules = defaultdict(dict)
5765
for each_file in self.__raw_files:
5866
each_filename = os.path.basename(each_file)

0 commit comments

Comments
 (0)