Skip to content

Commit

Permalink
Merge 9089b93 into 27406e3
Browse files Browse the repository at this point in the history
  • Loading branch information
@wphyojpl
wphyojpl authored Dec 19, 2022
2 parents 27406e3 + 9089b93 commit 22de442
Show file tree
Hide file tree
Showing 34 changed files with 1,184 additions and 34 deletions.
32 changes: 27 additions & 5 deletions cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,32 @@
import os

from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery
from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils

LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())


class CumulusCollectionsDapa:
RESOURCE = 'COLLECTIONS'
ACTION = 'READ'

def __init__(self, event):
LOGGER.info(f'event: {event}')
required_env = ['CUMULUS_BASE', 'CUMULUS_LAMBDA_PREFIX', 'COGNITO_UESR_POOL_ID', 'ES_URL']
if not all([k in os.environ for k in required_env]):
raise EnvironmentError(f'one or more missing env: {required_env}')

self.__event = event
self.__jwt_token = 'NA'
self.__limit = 10
self.__offset = 0
self.__assign_values()
self.__page_number = (self.__offset // self.__limit) + 1
if 'CUMULUS_BASE' not in os.environ:
raise EnvironmentError('missing key: CUMULUS_BASE')
if 'CUMULUS_LAMBDA_PREFIX' not in os.environ:
raise EnvironmentError('missing key: CUMULUS_LAMBDA_PREFIX')

self.__cumulus_base = os.getenv('CUMULUS_BASE')
self.__cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX')
Expand All @@ -29,6 +36,14 @@ def __init__(self, event):
self.__cumulus.with_limit(self.__limit)
self.__cumulus.with_page_number(self.__page_number)
self.__get_collection_id()
self.__lambda_utils = LambdaApiGatewayUtils(self.__event, self.__limit)
self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory().\
get_instance(UDSAuthorizerFactory.cognito,
user_pool_id=os.environ.get('COGNITO_UESR_POOL_ID'),
es_url=os.getenv('ES_URL'),
es_port=int(os.getenv('ES_PORT', '443'))
)
self.__uds_collections = UdsCollections(os.getenv('ES_URL'), int(os.getenv('ES_PORT', '443')))

def __get_collection_id(self):
if 'pathParameters' not in self.__event:
Expand Down Expand Up @@ -62,14 +77,21 @@ def __get_size(self):

def __get_pagination_urls(self):
try:
pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
pagination_links = self.__lambda_utils.generate_pagination_links()
except Exception as e:
LOGGER.exception(f'error while generating pagination links')
return [{'message': f'error while generating pagination links: {str(e)}'}]
return pagination_links

def start(self):
try:
ldap_groups = self.__lambda_utils.get_authorization_info()['ldap_groups']

collection_regexes = self.__authorizer.get_authorized_collections(DBConstants.read, ldap_groups)
authorized_collections = self.__uds_collections.get_collections(collection_regexes)
authorized_collection_ids = [k[DBConstants.collection_id] for k in authorized_collections]
# NOTE: 2022-11-21: only pass collections. not versions
self.__cumulus.with_collections(authorized_collection_ids)
cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
if 'server_error' in cumulus_result:
return {
Expand Down
55 changes: 53 additions & 2 deletions cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,20 @@

from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer
from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery
from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
from cumulus_lambda_functions.lib.aws.aws_lambda import AwsLambda
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils

LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())


class CumulusCreateCollectionDapa:
def __init__(self, event):
required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL']
required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL', 'COGNITO_UESR_POOL_ID', 'ES_URL']
if not all([k in os.environ for k in required_env]):
raise EnvironmentError(f'one or more missing env: {required_env}')
self.__event = event
Expand All @@ -23,7 +28,16 @@ def __init__(self, event):
self.__ingest_sqs_url = os.getenv('CUMULUS_WORKFLOW_SQS_URL')
self.__workflow_name = os.getenv('CUMULUS_WORKFLOW_NAME', 'CatalogGranule')
self.__provider_id = os.getenv('UNITY_DEFAULT_PROVIDER', '')
self.__es_url = os.getenv('ES_URL')
self.__es_port = int(os.getenv('ES_PORT', '443'))
self.__collection_creation_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip()
self.__lambda_utils = LambdaApiGatewayUtils(self.__event, 10)
self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory()\
.get_instance(UDSAuthorizerFactory.cognito,
user_pool_id=os.getenv('COGNITO_UESR_POOL_ID'),
es_url=self.__es_url,
es_port=self.__es_port
)

def execute_creation(self):
try:
Expand All @@ -39,6 +53,26 @@ def execute_creation(self):
'message': creation_result
})
}
uds_collection = UdsCollections(self.__es_url, self.__es_port)
try:
time_range = collection_transformer.get_collection_time_range()
uds_collection.add_collection(
collection_id=collection_transformer.get_collection_id(),
start_time=time_range[0], # TODO convert to timestamp
end_time=time_range[1], # TODO convert to timestamp
bbox=collection_transformer.get_collection_bbox(),
granules_count=0,
)
except Exception as e:
LOGGER.exception(f'failed to add collection to Elasticsearch')
delete_collection_result = self.__cumulus_collection_query.delete_collection(self.__cumulus_lambda_prefix, cumulus_collection_doc['name'], cumulus_collection_doc['version'])
return {
'statusCode': 500,
'body': json.dumps({
'message': f'unable to add collection to Elasticsearch: {str(e)}',
'details': f'collection deletion result: {delete_collection_result}'
})
}
LOGGER.debug(f'__provider_id: {self.__provider_id}')
rule_creation_result = self.__cumulus_collection_query.create_sqs_rules(
cumulus_collection_doc,
Expand All @@ -50,6 +84,7 @@ def execute_creation(self):
if 'status' not in rule_creation_result:
LOGGER.error(f'status not in rule_creation_result. deleting collection: {rule_creation_result}')
delete_collection_result = self.__cumulus_collection_query.delete_collection(self.__cumulus_lambda_prefix, cumulus_collection_doc['name'], cumulus_collection_doc['version'])
uds_collection.delete_collection(collection_transformer.get_collection_id())
return {
'statusCode': 500,
'body': json.dumps({
Expand Down Expand Up @@ -79,14 +114,30 @@ def start(self):
raise ValueError(f'missing body in {self.__event}')
self.__request_body = json.loads(self.__event['body'])
LOGGER.debug(f'request body: {self.__request_body}')
validation_result = pystac.Collection.from_dict(self.__request_body).validate()
stac_collection = pystac.Collection.from_dict(self.__request_body)
validation_result = stac_collection.validate()
if not isinstance(validation_result, list):
LOGGER.error(f'request body is not valid STAC collection: {validation_result}')
return {
'statusCode': 500,
'body': json.dumps({'message': f'request body is not valid STAC Collection schema. check details',
'details': validation_result})
}

auth_info = self.__lambda_utils.get_authorization_info()
collection_id = stac_collection.id
collection_identifier = UdsCollections.decode_identifier(collection_id)
LOGGER.debug(f'query for user: {auth_info["username"]}')
if not self.__authorizer.is_authorized_for_collection(DBConstants.create, collection_id, auth_info['ldap_groups'],
collection_identifier.tenant,
collection_identifier.venue):
LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}')
return {
'statusCode': 403,
'body': json.dumps({
'message': 'not authorized to create an action'
})
}
if self.__collection_creation_lambda_name != '':
response = AwsLambda().invoke_function(
function_name=self.__collection_creation_lambda_name,
Expand Down
54 changes: 43 additions & 11 deletions cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@
import os

from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery
from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils

LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())
Expand All @@ -20,17 +24,18 @@ def __init__(self, event):
:param event:
"""
LOGGER.info(f'event: {event}')
required_env = ['CUMULUS_BASE', 'CUMULUS_LAMBDA_PREFIX', 'COGNITO_UESR_POOL_ID', 'ES_URL']
if not all([k in os.environ for k in required_env]):
raise EnvironmentError(f'one or more missing env: {required_env}')


self.__event = event
self.__jwt_token = ''
self.__datetime = None
self.__limit = 10
self.__offset = 0
self.__assign_values()
self.__page_number = (self.__offset // self.__limit) + 1
if 'CUMULUS_BASE' not in os.environ:
raise EnvironmentError('missing key: CUMULUS_BASE')
if 'CUMULUS_LAMBDA_PREFIX' not in os.environ:
raise EnvironmentError('missing key: CUMULUS_LAMBDA_PREFIX')

self.__cumulus_base = os.getenv('CUMULUS_BASE')
self.__cumulus_lambda_prefix = os.getenv('CUMULUS_LAMBDA_PREFIX')
Expand All @@ -39,19 +44,26 @@ def __init__(self, event):
self.__cumulus.with_limit(self.__limit)
self.__cumulus.with_page_number(self.__page_number)
self.__get_time_range()
self.__get_collection_id()
self.__collection_id = self.__get_collection_id()
self.__cumulus.with_collection_id(self.__collection_id)
self.__lambda_utils = LambdaApiGatewayUtils(self.__event, self.__limit)
self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory()\
.get_instance(UDSAuthorizerFactory.cognito,
user_pool_id=os.environ.get('COGNITO_UESR_POOL_ID'),
es_url=os.getenv('ES_URL'),
es_port=int(os.getenv('ES_PORT', '443'))
)

def __get_collection_id(self):
if 'pathParameters' not in self.__event:
return self
return ''
path_param_dict = self.__event['pathParameters']
if 'collectionId' not in path_param_dict:
return self
return ''
collection_id = path_param_dict['collectionId']
if collection_id == '*':
return self
self.__cumulus.with_collection_id(path_param_dict['collectionId'])
return self
return ''
return path_param_dict['collectionId']

def __get_time_range(self):
if self.__datetime is None:
Expand Down Expand Up @@ -98,14 +110,34 @@ def __get_size(self):

def __get_pagination_urls(self):
try:
pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links()
pagination_links = self.__lambda_utils.generate_pagination_links()
except Exception as e:
LOGGER.exception(f'error while generating pagination links')
return [{'message': f'error while generating pagination links: {str(e)}'}]
return pagination_links

def start(self):
try:
if self.__collection_id == '':
return {
'statusCode': 500,
'body': json.dumps({'message': 'unknown collection_id. require 1 collection id. '})
}
auth_info = self.__lambda_utils.get_authorization_info()
collection_identifier = UdsCollections.decode_identifier(self.__collection_id)
if not self.__authorizer.is_authorized_for_collection(DBConstants.create, self.__collection_id, auth_info['ldap_groups'],
collection_identifier.tenant,
collection_identifier.venue):
LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {self.__collection_id}')
return {
'statusCode': 403,
'body': json.dumps({
'message': 'not authorized to create an action'
})
}

# TODO. cannot accept multiple collection_id. need single collection_id
# get project and project_venue from collection_id and compare against authorization table
cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix)
if 'server_error' in cumulus_result:
return {
Expand Down
36 changes: 34 additions & 2 deletions ...lus_lambda_functions/cumulus_granules_dapa_ingest_cnm/cumulus_granules_dapa_ingest_cnm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@
import os

from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery
from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract
from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory
from cumulus_lambda_functions.lib.aws.aws_sns import AwsSns
from cumulus_lambda_functions.lib.json_validator import JsonValidator
from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator
from cumulus_lambda_functions.lib.time_utils import TimeUtils
from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants
from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections
from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils

LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env())

Expand Down Expand Up @@ -46,11 +51,19 @@ def __init__(self, event):
:param event:
"""
LOGGER.debug(f'event: {event}')
if 'SNS_TOPIC_ARN' not in os.environ:
raise EnvironmentError('missing key: SNS_TOPIC_ARN')
required_env = ['SNS_TOPIC_ARN', 'COGNITO_UESR_POOL_ID', 'ES_URL']
if not all([k in os.environ for k in required_env]):
raise EnvironmentError(f'one or more missing env: {required_env}')
self.__event = event
self.__request_body = {}
self.__sns_topic_arn = os.getenv('SNS_TOPIC_ARN')
self.__lambda_utils = LambdaApiGatewayUtils(self.__event, 10)
self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory()\
.get_instance(UDSAuthorizerFactory.cognito,
user_pool_id=os.getenv('COGNITO_UESR_POOL_ID'),
es_url=os.getenv('ES_URL'),
es_port=int(os.getenv('ES_PORT', '443'))
)

def __get_json_request_body(self):
if 'body' not in self.__event:
Expand Down Expand Up @@ -156,6 +169,25 @@ def start(self):
:return:
"""
self.__get_json_request_body()
collection_ids = list(set([k['collection'] for k in self.__request_body['features']]))
if len(collection_ids) != 1:
return {
'statusCode': 500,
'body': json.dumps({'message': f'does not allow multiple collections in a single request', 'details': collection_ids})
}
auth_info = self.__lambda_utils.get_authorization_info()
collection_id = collection_ids[0]
collection_identifier = UdsCollections.decode_identifier(collection_id)
if not self.__authorizer.is_authorized_for_collection(DBConstants.create, collection_id, auth_info['ldap_groups'],
collection_identifier.tenant,
collection_identifier.venue):
LOGGER.debug(f'user: {auth_info["username"]} is not authorized for {collection_id}')
return {
'statusCode': 403,
'body': json.dumps({
'message': 'not authorized to create an action'
})
}
error_list = []
for each_granule in self.__request_body['features']:
LOGGER.debug(f'executing: {each_granule}')
Expand Down
Loading

0 comments on commit 22de442

Please sign in to comment.