Skip to content

Commit 0b65741

Browse files
authored
Merge e427432 into 27406e3
2 parents 27406e3 + e427432 commit 0b65741

40 files changed

+1304
-34
lines changed

ci.cd/Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ update_lambda_function_4:
2929
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa --publish &>/dev/null
3030
update_lambda_function_5:
3131
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_collections_creation_dapa_facade --publish &>/dev/null
32+
update_lambda_function_6:
33+
aws --profile saml-pub lambda update-function-code --s3-key unity_cumulus_lambda/cumulus_lambda_functions_deployment.zip --s3-bucket am-uds-dev-cumulus-tf-state --function-name arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-cumulus_es_setup_index_alias --publish &>/dev/null
3234

35+
jpl_cloud_test_1: build_lambda upload_lambda update_lambda_function_6
3336

3437
upload_lambda_mcp_dev:
3538
aws s3 cp cumulus_lambda_functions_deployment.zip s3://uds-dev-cumulus-public/unity_cumulus_lambda/

cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,32 @@
22
import os
33

44
from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery
5+
from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
6+
from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
57
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
8+
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
9+
from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
610
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils
711

812
LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
913

1014

1115
class CumulusCollectionsDapa:
16+
RESOURCE = 'COLLECTIONS'
17+
ACTION = 'READ'
18+
1219
def __init__(self, event):
1320
LOGGER.info(f'event: {event}')
21+
required_env = ['CUMULUS_BASE', 'CUMULUS_LAMBDA_PREFIX', 'COGNITO_UESR_POOL_ID', 'ES_URL']
22+
if not all([k in os.environ for k in required_env]):
23+
raise EnvironmentError(f'one or more missing env: {required_env}')
24+
1425
self.__event = event
1526
self.__jwt_token = 'NA'
1627
self.__limit = 10
1728
self.__offset = 0
1829
self.__assign_values()
1930
self.__page_number = (self.__offset // self.__limit) + 1
20-
if 'CUMULUS_BASE' not in os.environ:
21-
raise EnvironmentError('missing key: CUMULUS_BASE')
22-
if 'CUMULUS_LAMBDA_PREFIX' not in os.environ:
23-
raise EnvironmentError('missing key: CUMULUS_LAMBDA_PREFIX')
2431

2532
self.__cumulus_base = os.getenv('CUMULUS_BASE')
2633
self.__cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX')
@@ -29,6 +36,14 @@ def __init__(self, event):
2936
self.__cumulus.with_limit(self.__limit)
3037
self.__cumulus.with_page_number(self.__page_number)
3138
self.__get_collection_id()
39+
self.__lambda_utils = LambdaApiGatewayUtils(self.__event, self.__limit)
40+
self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory().\
41+
get_instance(UDSAuthorizerFactory.cognito,
42+
user_pool_id=os.environ.get('COGNITO_UESR_POOL_ID'),
43+
es_url=os.getenv('ES_URL'),
44+
es_port=int(os.getenv('ES_PORT', '443'))
45+
)
46+
self.__uds_collections = UdsCollections(os.getenv('ES_URL'), int(os.getenv('ES_PORT', '443')))
3247

3348
def __get_collection_id(self):
3449
if 'pathParameters' not in self.__event:
@@ -62,14 +77,21 @@ def __get_size(self):
6277

6378
def __get_pagination_urls(self):
6479
try:
65-
pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
80+
pagination_links = self.__lambda_utils.generate_pagination_links()
6681
except Exception as e:
6782
LOGGER.exception(f'error while generating pagination links')
6883
return [{'message': f'error while generating pagination links: {str(e)}'}]
6984
return pagination_links
7085

7186
def start(self):
7287
try:
88+
ldap_groups = self.__lambda_utils.get_authorization_info()['ldap_groups']
89+
90+
collection_regexes = self.__authorizer.get_authorized_collections(DBConstants.read, ldap_groups)
91+
authorized_collections = self.__uds_collections.get_collections(collection_regexes)
92+
authorized_collection_ids = [k[DBConstants.collection_id] for k in authorized_collections]
93+
# NOTE: 2022-11-21: only pass collections. not versions
94+
self.__cumulus.with_collections(authorized_collection_ids)
7395
cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
7496
if 'server_error' in cumulus_result:
7597
return {

cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,20 @@
55

66
from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer
77
from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery
8+
from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
9+
from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
810
from cumulus_lambda_functions.lib.aws.aws_lambda import AwsLambda
911
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
12+
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
13+
from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
14+
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils
1015

1116
LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
1217

1318

1419
class CumulusCreateCollectionDapa:
1520
def __init__(self, event):
16-
required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL']
21+
required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL', 'COGNITO_UESR_POOL_ID', 'ES_URL']
1722
if not all([k in os.environ for k in required_env]):
1823
raise EnvironmentError(f'one or more missing env: {required_env}')
1924
self.__event = event
@@ -23,7 +28,16 @@ def __init__(self, event):
2328
self.__ingest_sqs_url = os.getenv('CUMULUS_WORKFLOW_SQS_URL')
2429
self.__workflow_name = os.getenv('CUMULUS_WORKFLOW_NAME', 'CatalogGranule')
2530
self.__provider_id = os.getenv('UNITY_DEFAULT_PROVIDER', '')
31+
self.__es_url = os.getenv('ES_URL')
32+
self.__es_port = int(os.getenv('ES_PORT', '443'))
2633
self.__collection_creation_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip()
34+
self.__lambda_utils = LambdaApiGatewayUtils(self.__event, 10)
35+
self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory()\
36+
.get_instance(UDSAuthorizerFactory.cognito,
37+
user_pool_id=os.getenv('COGNITO_UESR_POOL_ID'),
38+
es_url=self.__es_url,
39+
es_port=self.__es_port
40+
)
2741

2842
def execute_creation(self):
2943
try:
@@ -39,6 +53,26 @@ def execute_creation(self):
3953
'message': creation_result
4054
})
4155
}
56+
uds_collection = UdsCollections(self.__es_url, self.__es_port)
57+
try:
58+
time_range = collection_transformer.get_collection_time_range()
59+
uds_collection.add_collection(
60+
collection_id=collection_transformer.get_collection_id(),
61+
start_time=time_range[0], # TODO convert to timestamp
62+
end_time=time_range[1], # TODO convert to timestamp
63+
bbox=collection_transformer.get_collection_bbox(),
64+
granules_count=0,
65+
)
66+
except Exception as e:
67+
LOGGER.exception(f'failed to add collection to Elasticsearch')
68+
delete_collection_result = self.__cumulus_collection_query.delete_collection(self.__cumulus_lambda_prefix, cumulus_collection_doc['name'], cumulus_collection_doc['version'])
69+
return {
70+
'statusCode': 500,
71+
'body': json.dumps({
72+
'message': f'unable to add collection to Elasticsearch: {str(e)}',
73+
'details': f'collection deletion result: {delete_collection_result}'
74+
})
75+
}
4276
LOGGER.debug(f'__provider_id: {self.__provider_id}')
4377
rule_creation_result = self.__cumulus_collection_query.create_sqs_rules(
4478
cumulus_collection_doc,
@@ -50,6 +84,7 @@ def execute_creation(self):
5084
if 'status' not in rule_creation_result:
5185
LOGGER.error(f'status not in rule_creation_result. deleting collection: {rule_creation_result}')
5286
delete_collection_result = self.__cumulus_collection_query.delete_collection(self.__cumulus_lambda_prefix, cumulus_collection_doc['name'], cumulus_collection_doc['version'])
87+
uds_collection.delete_collection(collection_transformer.get_collection_id())
5388
return {
5489
'statusCode': 500,
5590
'body': json.dumps({
@@ -79,14 +114,30 @@ def start(self):
79114
raise ValueError(f'missing body in {self.__event}')
80115
self.__request_body = json.loads(self.__event['body'])
81116
LOGGER.debug(f'request body: {self.__request_body}')
82-
validation_result = pystac.Collection.from_dict(self.__request_body).validate()
117+
stac_collection = pystac.Collection.from_dict(self.__request_body)
118+
validation_result = stac_collection.validate()
83119
if not isinstance(validation_result, list):
84120
LOGGER.error(f'request body is not valid STAC collection: {validation_result}')
85121
return {
86122
'statusCode': 500,
87123
'body': json.dumps({'message': f'request body is not valid STAC Collection schema. check details',
88124
'details': validation_result})
89125
}
126+
127+
auth_info = self.__lambda_utils.get_authorization_info()
128+
collection_id = stac_collection.id
129+
collection_identifier = UdsCollections.decode_identifier(collection_id)
130+
LOGGER.debug(f'query for user: {auth_info["username"]}')
131+
if not self.__authorizer.is_authorized_for_collection(DBConstants.create, collection_id, auth_info['ldap_groups'],
132+
collection_identifier.tenant,
133+
collection_identifier.venue):
134+
LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}')
135+
return {
136+
'statusCode': 403,
137+
'body': json.dumps({
138+
'message': 'not authorized to create an action'
139+
})
140+
}
90141
if self.__collection_creation_lambda_name != '':
91142
response = AwsLambda().invoke_function(
92143
function_name=self.__collection_creation_lambda_name,

cumulus_lambda_functions/cumulus_es_setup/__init__.py

Whitespace-only changes.
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
alias_pointer = {
2+
"actions" : [
3+
{"add" : {"index" : "authorization_mappings_v1", "alias" : "authorization_mappings"}},
4+
{"add" : {"index" : "unity_collections_v1", "alias" : "unity_collections"}}
5+
]
6+
}
7+
8+
authorization_mappings_v1 = {
9+
"settings" : {
10+
"number_of_shards" : 3,
11+
"number_of_replicas" : 2
12+
},
13+
"mappings": {
14+
"properties": {
15+
"action": {"type": "keyword"},
16+
"collection_map": {"type": "keyword"},
17+
"user_group": {"type": "keyword"},
18+
"tenant": {"type": "keyword"},
19+
"tenant_venue": {"type": "keyword"}
20+
}
21+
}
22+
}
23+
24+
unity_collections_v1 = {
25+
"settings" : {
26+
"number_of_shards" : 3,
27+
"number_of_replicas" : 2
28+
},
29+
"mappings": {
30+
"properties": {
31+
"collection_id": {"type": "keyword"},
32+
"bbox": {"type": "geo_shape"},
33+
"granule_count": {"type": "integer"},
34+
"start_time": {"type": "long"},
35+
"end_time": {"type": "long"}
36+
}
37+
}
38+
}
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import os
2+
3+
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
4+
5+
from cumulus_lambda_functions.lib.aws.es_abstract import ESAbstract
6+
7+
from cumulus_lambda_functions.lib.aws.es_factory import ESFactory
8+
9+
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
10+
from cumulus_lambda_functions.cumulus_es_setup import es_mappings
11+
LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
12+
13+
14+
class SetupESIndexAlias:
15+
def __init__(self):
16+
required_env = ['ES_URL']
17+
if not all([k in os.environ for k in required_env]):
18+
raise EnvironmentError(f'one or more missing env: {required_env}')
19+
self.__es: ESAbstract = ESFactory().get_instance('AWS',
20+
index=DBConstants.collections_index,
21+
base_url=os.getenv('ES_URL'),
22+
port=int(os.getenv('ES_PORT', '443'))
23+
)
24+
25+
def get_index_mapping(self, index_name: str):
26+
if not hasattr(es_mappings, index_name):
27+
raise ValueError(f'missing index_name: {index_name}')
28+
index_json = getattr(es_mappings, index_name)
29+
return index_json
30+
31+
def start(self):
32+
if not hasattr(es_mappings, 'alias_pointer'):
33+
raise ValueError(f'missing alias_pointer')
34+
alias_json = getattr(es_mappings, 'alias_pointer')
35+
alias_json = [k['add'] for k in alias_json['actions']]
36+
for each_action in alias_json:
37+
current_index = each_action['index']
38+
current_alias = each_action['alias']
39+
LOGGER.debug(f'working on {current_index}')
40+
index_json = self.get_index_mapping(current_index)
41+
self.__es.create_index(current_index, index_json)
42+
self.__es.create_alias(current_index, current_alias)
43+
return self
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from cumulus_lambda_functions.cumulus_es_setup.es_setup import SetupESIndexAlias
2+
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
3+
4+
5+
def lambda_handler(event, context):
6+
"""
7+
{'cma': {'task_config': {'bucket': '{$.meta.buckets.internal.name}', 'collection': '{$.meta.collection}', 'cumulus_message': {'outputs': [{'source': '{$.files}', 'destination': '{$.payload}'}]}}, 'event': {'cumulus_meta': {'cumulus_version': '10.0.1', 'execution_name': 'c6d885dc-b4b2-4eb0-b22e-b6f58a7a0870', 'message_source': 'sfn', 'queueExecutionLimits': {'https://sqs.us-west-2.amazonaws.com/884500545225/am-uds-dev-cumulus-backgroundProcessing': 5}, 'state_machine': 'arn:aws:states:us-west-2:884500545225:stateMachine:am-uds-dev-cumulus-IngestGranule', 'system_bucket': 'am-uds-dev-cumulus-internal', 'workflow_start_time': 1646785175509, 'parentExecutionArn': 'arn:aws:states:us-west-2:884500545225:execution:am-uds-dev-cumulus-DiscoverGranules:885483b4-ba55-4db1-b197-661e1e595a45', 'queueUrl': 'arn:aws:sqs:us-west-2:884500545225:am-uds-dev-cumulus-startSF'}, 'exception': 'None', 'meta': {'buckets': {'internal': {'name': 'am-uds-dev-cumulus-internal', 'type': 'internal'}, 'protected': {'name': 'am-uds-dev-cumulus-protected', 'type': 'protected'}}, 'cmr': {'clientId': 'CHANGEME', 'cmrEnvironment': 'UAT', 'cmrLimit': 100, 'cmrPageSize': 50, 'oauthProvider': 'earthdata', 'passwordSecretName': 'am-uds-dev-cumulus-message-template-cmr-password20220216072916956000000002', 'provider': 'CHANGEME', 'username': 'username'}, 'collection': {'name': 'ATMS_SCIENCE_Group_2011', 'version': '001', 'process': 'modis', 'granuleId': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0$', 'granuleIdExtraction': '(P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0).+', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS', 'duplicateHandling': 'replace', 'url_path': '{cmrMetadata.Granule.Collection.ShortName}___{cmrMetadata.Granule.Collection.VersionId}', 'provider_path': '/data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/', 'files': [{'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}00\\.PDS$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000000.PDS', 'type': 'data'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS', 'type': 'metadata'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}01\\.PDS\\.xml$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS.xml', 'type': 'metadata'}, {'bucket': 'internal', 'regex': '^P[0-9]{3}[0-9]{4}[A-Z]{13}T[0-9]{12}0\\.cmr\\.xml$', 'sampleFileName': 'P1570515ATMSSCIENCEAXT11344000000001.PDS.xml', 'type': 'metadata'}], 'updatedAt': 1646326197526, 'createdAt': 1646258167624}, 'distribution_endpoint': 's3://am-uds-dev-cumulus-internal/', 'launchpad': {'api': 'launchpadApi', 'certificate': 'launchpad.pfx', 'passphraseSecretName': ''}, 'provider': {'password': 'AQICAHhSagsGDAl5tQWM010IEvxKgj2LcsNub5v5FHoRpOjXcQHFbE4iMnF/W0Y/NrsYvrfHAAAAajBoBgkqhkiG9w0BBwagWzBZAgEAMFQGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMLaH13SdxPXREjXLtAgEQgCfA+lEu2c/xLTGwJsbtKlXJbKDy4pwV+rS3BnJqgBoLLMQZqOdoFhk=', 'host': 'snppl0.gesdisc.eosdis.nasa.gov', 'updatedAt': 1646244053419, 'protocol': 'https', 'createdAt': 1646244053419, 'encrypted': True, 'username': 'AQICAHhSagsGDAl5tQWM010IEvxKgj2LcsNub5v5FHoRpOjXcQGRoY5EBMpvvyMASUowBM61AAAAYzBhBgkqhkiG9w0BBwagVDBSAgEAME0GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM9OhRHwTuxiz74q4UAgEQgCDEHOhsVG6+LqXfnlw+Z3Wg9MDOCd9/K5/X5j3tPJYkaA==', 'allowedRedirects': ['https://urs.earthdata.nasa.gov', 'urs.earthdata.nasa.gov'], 'id': 'snpp_provider_02', 'globalConnectionLimit': 10}, 'stack': 'am-uds-dev-cumulus', 'template': 's3://am-uds-dev-cumulus-internal/am-uds-dev-cumulus/workflow_template.json', 'workflow_name': 'IngestGranule', 'workflow_tasks': {'SyncGranule': {'name': 'am-uds-dev-cumulus-SyncGranule', 'version': '$LATEST', 'arn': 'arn:aws:lambda:us-west-2:884500545225:function:am-uds-dev-cumulus-SyncGranule'}}, 'staticValue': 'aStaticValue', 'interpolatedValueStackName': 'am-uds-dev-cumulus', 'input_granules': [{'granuleId': 'P1570515ATMSSCIENCEAXT1134912000000', 'dataType': 'ATMS_SCIENCE_Group_2011', 'version': '001', 'files': [{'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000000.PDS', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000000.PDS', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000000.PDS', 'type': 'data', 'size': 744}, {'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000001.PDS', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000001.PDS', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000001.PDS', 'type': 'metadata', 'size': 18084600}, {'bucket': 'am-uds-dev-cumulus-internal', 'key': 'file-staging/am-uds-dev-cumulus/ATMS_SCIENCE_Group_2011___001/P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'source': 'data/SNPP_ATMS_Level0_T/ATMS_SCIENCE_Group/2011/349//P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'fileName': 'P1570515ATMSSCIENCEAXT11349120000001.PDS.xml', 'type': 'metadata', 'size': 9526}], 'sync_granule_duration': 9822, 'createdAt': 1647386972717}], 'process': 'modis'}, 'payload': {}, 'replace': {'Bucket': 'am-uds-dev-cumulus-internal', 'Key': 'events/5d8edf37-0a18-4af5-a76f-7c2091cdd1e2', 'TargetPath': '$.payload'}}}}
8+
:param event:
9+
:param context:
10+
:return:
11+
"""
12+
LambdaLoggerGenerator.remove_default_handlers()
13+
return SetupESIndexAlias().start()

0 commit comments

Comments
 (0)