diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py index cd91a4f2..c572f358 100644 --- a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py +++ b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_collections_dapa.py @@ -2,13 +2,19 @@ import os from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery +from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract +from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator +from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) class CumulusCollectionsDapa: + RESOURCE = 'COLLECTIONS' + ACTION = 'READ' + def __init__(self, event): LOGGER.info(f'event: {event}') self.__event = event @@ -17,6 +23,8 @@ def __init__(self, event): self.__offset = 0 self.__assign_values() self.__page_number = (self.__offset // self.__limit) + 1 + if 'COGNITO_UESR_POOL_ID' not in os.environ: + raise EnvironmentError('missing key: COGNITO_UESR_POOL_ID') if 'CUMULUS_BASE' not in os.environ: raise EnvironmentError('missing key: CUMULUS_BASE') if 'CUMULUS_LAMBDA_PREFIX' not in os.environ: @@ -29,6 +37,8 @@ def __init__(self, event): self.__cumulus.with_limit(self.__limit) self.__cumulus.with_page_number(self.__page_number) self.__get_collection_id() + self.__lambda_utils = LambdaApiGatewayUtils(self.__event, self.__limit) + self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory().get_instance(UDSAuthorizerFactory.cognito, user_pool_id=os.environ.get('COGNITO_UESR_POOL_ID')) def __get_collection_id(self): if 'pathParameters' not in self.__event: @@ -62,14 +72,23 @@ def __get_size(self): def __get_pagination_urls(self): try: - pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links() + pagination_links = self.__lambda_utils.generate_pagination_links() except Exception as e: LOGGER.exception(f'error while generating pagination links') return [{'message': f'error while generating pagination links: {str(e)}'}] return pagination_links + def __setup_authorized_tenant_venue(self): + username = self.__lambda_utils.get_authorization_info()['username'] + LOGGER.debug(f'query for user: {username}') + authorized_tenants = self.__authorizer.get_authorized_tenant(username, self.ACTION, self.RESOURCE) + for each_tenant in authorized_tenants: + self.__cumulus.with_tenant(each_tenant[DBConstants.tenant], each_tenant[DBConstants.tenant_venue]) + return self + def start(self): try: + self.__setup_authorized_tenant_venue() cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix) if 'server_error' in cumulus_result: return { diff --git a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py index abbdbecb..1c91c988 100644 --- a/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py +++ b/cumulus_lambda_functions/cumulus_collections_dapa/cumulus_create_collection_dapa.py @@ -5,15 +5,21 @@ from cumulus_lambda_functions.cumulus_stac.collection_transformer import CollectionTransformer from cumulus_lambda_functions.cumulus_wrapper.query_collections import CollectionsQuery +from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract +from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory from cumulus_lambda_functions.lib.aws.aws_lambda import AwsLambda from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator +from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) class CumulusCreateCollectionDapa: + RESOURCE = 'COLLECTIONS' + ACTION = 'WRITE' + def __init__(self, event): - required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL'] + required_env = ['CUMULUS_LAMBDA_PREFIX', 'CUMULUS_WORKFLOW_SQS_URL', 'COGNITO_UESR_POOL_ID'] if not all([k in os.environ for k in required_env]): raise EnvironmentError(f'one or more missing env: {required_env}') self.__event = event @@ -24,6 +30,8 @@ def __init__(self, event): self.__workflow_name = os.getenv('CUMULUS_WORKFLOW_NAME', 'CatalogGranule') self.__provider_id = '' # TODO. need this? self.__collection_creation_lambda_name = os.environ.get('COLLECTION_CREATION_LAMBDA_NAME', '').strip() + self.__lambda_utils = LambdaApiGatewayUtils(self.__event, 10) + self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory().get_instance(UDSAuthorizerFactory.cognito, user_pool_id=os.environ.get('COGNITO_UESR_POOL_ID')) def execute_creation(self): try: @@ -74,6 +82,11 @@ def execute_creation(self): def start(self): if 'body' not in self.__event: raise ValueError(f'missing body in {self.__event}') + username = self.__lambda_utils.get_authorization_info()['username'] + LOGGER.debug(f'query for user: {username}') + authorized_tenants = self.__authorizer.get_authorized_tenant(username, self.ACTION, self.RESOURCE) + # TODO compare project and project_venue vs the ones from request body. + # TODO project and project_venue in STAC body? self.__request_body = json.loads(self.__event['body']) LOGGER.debug(f'request body: {self.__request_body}') validation_result = pystac.Collection.from_dict(self.__request_body).validate() diff --git a/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py b/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py index ca13b8cc..86617295 100644 --- a/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py +++ b/cumulus_lambda_functions/cumulus_granules_dapa/cumulus_granules_dapa.py @@ -2,13 +2,19 @@ import os from cumulus_lambda_functions.cumulus_wrapper.query_granules import GranulesQuery +from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract +from cumulus_lambda_functions.lib.authorization.uds_authorizer_factory import UDSAuthorizerFactory from cumulus_lambda_functions.lib.lambda_logger_generator import LambdaLoggerGenerator +from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants from cumulus_lambda_functions.lib.utils.lambda_api_gateway_utils import LambdaApiGatewayUtils LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) class CumulusGranulesDapa: + RESOURCE = 'GRANULES' + ACTION = 'READ' + def __init__(self, event): """ {'resource': '/collections/observation/items', @@ -27,6 +33,8 @@ def __init__(self, event): self.__offset = 0 self.__assign_values() self.__page_number = (self.__offset // self.__limit) + 1 + if 'COGNITO_UESR_POOL_ID' not in os.environ: + raise EnvironmentError('missing key: COGNITO_UESR_POOL_ID') if 'CUMULUS_BASE' not in os.environ: raise EnvironmentError('missing key: CUMULUS_BASE') if 'CUMULUS_LAMBDA_PREFIX' not in os.environ: @@ -40,6 +48,8 @@ def __init__(self, event): self.__cumulus.with_page_number(self.__page_number) self.__get_time_range() self.__get_collection_id() + self.__lambda_utils = LambdaApiGatewayUtils(self.__event, self.__limit) + self.__authorizer: UDSAuthorizorAbstract = UDSAuthorizerFactory().get_instance(UDSAuthorizerFactory.cognito, user_pool_id=os.environ.get('COGNITO_UESR_POOL_ID')) def __get_collection_id(self): if 'pathParameters' not in self.__event: @@ -98,14 +108,25 @@ def __get_size(self): def __get_pagination_urls(self): try: - pagination_links = LambdaApiGatewayUtils(self.__event, self.__limit).generate_pagination_links() + pagination_links = self.__lambda_utils.generate_pagination_links() except Exception as e: LOGGER.exception(f'error while generating pagination links') return [{'message': f'error while generating pagination links: {str(e)}'}] return pagination_links + def __setup_authorized_tenant_venue(self): + username = self.__lambda_utils.get_authorization_info()['username'] + LOGGER.debug(f'query for user: {username}') + authorized_tenants = self.__authorizer.get_authorized_tenant(username, self.ACTION, self.RESOURCE) + for each_tenant in authorized_tenants: + self.__cumulus.with_tenant(each_tenant[DBConstants.tenant], each_tenant[DBConstants.tenant_venue]) + return self + def start(self): try: + self.__setup_authorized_tenant_venue() + # TODO. cannot accept multiple collection_id. need single collection_id + # get project and project_venue from collection_id and compare against authorization table cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix) if 'server_error' in cumulus_result: return { diff --git a/cumulus_lambda_functions/cumulus_wrapper/cumulus_base.py b/cumulus_lambda_functions/cumulus_wrapper/cumulus_base.py index 30667bd7..041ee70a 100644 --- a/cumulus_lambda_functions/cumulus_wrapper/cumulus_base.py +++ b/cumulus_lambda_functions/cumulus_wrapper/cumulus_base.py @@ -13,6 +13,14 @@ def __init__(self, cumulus_base: str, cumulus_token: str): 'Authorization': f'Bearer {cumulus_token}' } self._conditions = ['status=completed'] + self._authorized_tenants = [] + + def with_tenant(self, tenant: str, venue: str): + self._authorized_tenants.append({ + 'tenant': tenant, + 'venue': venue, + }) + return self def with_page_number(self, page_number): self._conditions.append(f'page={page_number}') diff --git a/cumulus_lambda_functions/lib/authorization/__init__.py b/cumulus_lambda_functions/lib/authorization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cumulus_lambda_functions/lib/authorization/uds_authorizer_abstract.py b/cumulus_lambda_functions/lib/authorization/uds_authorizer_abstract.py new file mode 100644 index 00000000..be9b519d --- /dev/null +++ b/cumulus_lambda_functions/lib/authorization/uds_authorizer_abstract.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod + + +class UDSAuthorizorAbstract(ABC): + @abstractmethod + def add_authorized_group(self, action: [str], resource: [str], tenant: str, venue: str, ldap_group_name: str): + return + + @abstractmethod + def delete_authorized_group(self, tenant: str, venue: str, ldap_group_name: str): + return + + @abstractmethod + def list_authorized_groups_for(self, tenant: str, venue: str): + return + + @abstractmethod + def update_authorized_group(self, action: [str], resource: [str], tenant: str, venue: str, ldap_group_name: str): + return + + @abstractmethod + def get_authorized_tenant(self, username: str, action: str, resource: str) -> list: + return [] diff --git a/cumulus_lambda_functions/lib/authorization/uds_authorizer_es_identity_pool.py b/cumulus_lambda_functions/lib/authorization/uds_authorizer_es_identity_pool.py new file mode 100644 index 00000000..6c7b5608 --- /dev/null +++ b/cumulus_lambda_functions/lib/authorization/uds_authorizer_es_identity_pool.py @@ -0,0 +1,121 @@ +import logging +import os + +from cumulus_lambda_functions.lib.authorization.uds_authorizer_abstract import UDSAuthorizorAbstract +from cumulus_lambda_functions.lib.aws.aws_cognito import AwsCognito +from cumulus_lambda_functions.lib.aws.es_abstract import ESAbstract +from cumulus_lambda_functions.lib.aws.es_factory import ESFactory +from cumulus_lambda_functions.lib.uds_db.db_constants import DBConstants + +LOGGER = logging.getLogger(__name__) + + +class UDSAuthorizorEsIdentityPool(UDSAuthorizorAbstract): + + def __init__(self, user_pool_id: str) -> None: + super().__init__() + es_url = os.getenv('ES_URL') # TODO validation + self.__authorization_index = os.getenv('AUTHORIZATION_INDEX') # LDAP_Group_Permission + es_port = int(os.getenv('ES_PORT', '443')) + self.__cognito = AwsCognito(user_pool_id) + self.__es: ESAbstract = ESFactory().get_instance('AWS', + index=self.__authorization_index, + base_url=es_url, + port=es_port) + + def add_authorized_group(self, action: [str], resource: [str], tenant: str, venue: str, ldap_group_name: str): + self.__es.index_one({ + DBConstants.action_key: action, + DBConstants.resource_key: resource, + DBConstants.tenant: tenant, + DBConstants.tenant_venue: venue, + DBConstants.authorized_group_name_key: ldap_group_name, + }, f'{tenant}__{venue}__{ldap_group_name}', self.__authorization_index) + return + + def delete_authorized_group(self, tenant: str, venue: str, ldap_group_name: str): + self.__es.delete_by_query({ + 'query': { + 'bool': { + 'must': [ + {'term': {DBConstants.tenant: tenant}}, + {'term': {DBConstants.tenant_venue: venue}}, + {'term': {DBConstants.authorized_group_name_key: ldap_group_name}}, + ] + } + } + }) + return + + def list_authorized_groups_for(self, tenant: str, venue: str): + result = self.__es.query_pages({ + 'query': { + 'bool': { + 'must': [ + {'term': {DBConstants.tenant: tenant}}, + {'term': {DBConstants.tenant_venue: venue}}, + ] + } + }, + 'sort': [ + {DBConstants.tenant: {'order': 'asc'}}, + {DBConstants.tenant_venue: {'order': 'asc'}}, + {DBConstants.authorized_group_name_key: {'order': 'asc'}}, + ] + }) + result = [k['_source'] for k in result['hits']['hits']] + return result + + def update_authorized_group(self, action: [str], resource: [str], tenant: str, venue: str, ldap_group_name: str): + self.__es.update_one({ + DBConstants.action_key: action, + DBConstants.resource_key: resource, + DBConstants.tenant: tenant, + DBConstants.tenant_venue: venue, + DBConstants.authorized_group_name_key: ldap_group_name, + }, f'{tenant}__{venue}__{ldap_group_name}', self.__authorization_index) + return + + def get_authorized_tenant(self, username: str, action: str, resource: str) -> list: + belonged_groups = set(self.__cognito.get_groups(username)) + + authorized_groups = self.__es.query({ + 'query': { + 'bool': { + 'must': [ + { + 'terms': { + DBConstants.authorized_group_name_key: list(belonged_groups), + } + }, + { + 'term': { + DBConstants.action_key: action, + } + }, + { + 'term': { + DBConstants.resource_key: resource, + } + } + ] + } + } + }) + return [k['_source'] for k in authorized_groups['hits']['hits']] + + def authorize(self, username, resource, action) -> bool: + belonged_groups = set(self.__cognito.get_groups(username)) + authorized_groups = self.__es.query({ + 'query': { + 'match_all': {} # TODO + } + }) + LOGGER.debug(f'belonged_groups for {username}: {belonged_groups}') + authorized_groups = set([k['_source']['group_name'] for k in authorized_groups['hits']['hits']]) + LOGGER.debug(f'authorized_groups for {resource}-{action}: {authorized_groups}') + if any([k in authorized_groups for k in belonged_groups]): + LOGGER.debug(f'{username} is authorized for {resource}-{action}') + return True + LOGGER.debug(f'{username} is NOT authorized for {resource}-{action}') + return False diff --git a/cumulus_lambda_functions/lib/authorization/uds_authorizer_factory.py b/cumulus_lambda_functions/lib/authorization/uds_authorizer_factory.py new file mode 100644 index 00000000..539d9528 --- /dev/null +++ b/cumulus_lambda_functions/lib/authorization/uds_authorizer_factory.py @@ -0,0 +1,12 @@ +from cumulus_lambda_functions.lib.aws.factory_abstract import FactoryAbstract + + +class UDSAuthorizerFactory(FactoryAbstract): + cognito = 'COGNITO' + + def get_instance(self, class_type, **kwargs): + if class_type == self.cognito: + from cumulus_lambda_functions.lib.authorization.uds_authorizer_es_identity_pool import \ + UDSAuthorizorEsIdentityPool + return UDSAuthorizorEsIdentityPool(kwargs['user_pool_id']) + raise ValueError(f'class_type: {class_type} not implemented') diff --git a/cumulus_lambda_functions/lib/aws/aws_cognito.py b/cumulus_lambda_functions/lib/aws/aws_cognito.py new file mode 100644 index 00000000..b31804f5 --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/aws_cognito.py @@ -0,0 +1,20 @@ +from cumulus_lambda_functions.lib.aws.aws_cred import AwsCred + + +class AwsCognito(AwsCred): + def __init__(self, user_pool_id: str): + super().__init__() + self.__cognito = self.get_client('cognito-idp') + self.__user_pool_id = user_pool_id + + def get_groups(self, username: str): + response = self.__cognito.admin_list_groups_for_user( + Username=username, + UserPoolId=self.__user_pool_id, + Limit=60, + # NextToken='string' + ) + if response is None or 'Groups' not in response: + return [] + belonged_groups = [k['GroupName'] for k in response['Groups']] + return belonged_groups diff --git a/cumulus_lambda_functions/lib/aws/aws_cred.py b/cumulus_lambda_functions/lib/aws/aws_cred.py index 761e73c4..7a4be33b 100644 --- a/cumulus_lambda_functions/lib/aws/aws_cred.py +++ b/cumulus_lambda_functions/lib/aws/aws_cred.py @@ -43,6 +43,19 @@ def __init__(self): else: LOGGER.debug('using default session as there is no aws_access_key_id') + @property + def region(self): + return self.__region + + @region.setter + def region(self, val): + """ + :param val: + :return: None + """ + self.__region = val + return + @property def boto3_session(self): return self.__boto3_session @@ -56,6 +69,9 @@ def boto3_session(self, val): self.__boto3_session = val return + def get_session(self): + return boto3.Session(**self.boto3_session) + def get_resource(self, service_name: str): return boto3.Session(**self.boto3_session).resource(service_name) diff --git a/cumulus_lambda_functions/lib/aws/es_abstract.py b/cumulus_lambda_functions/lib/aws/es_abstract.py new file mode 100644 index 00000000..fa82d957 --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/es_abstract.py @@ -0,0 +1,63 @@ +from abc import ABC, abstractmethod +from typing import Any, Union, Callable + +DEFAULT_TYPE = '_doc' + + +class ESAbstract(ABC): + @abstractmethod + def create_index(self, index_name, index_body): + return + + @abstractmethod + def has_index(self, index_name): + return + + @abstractmethod + def create_alias(self, index_name, alias_name): + return + + @abstractmethod + def delete_index(self, index_name): + return + + @abstractmethod + def index_many(self, docs=None, doc_ids=None, doc_dict=None, index=None): + return + + @abstractmethod + def index_one(self, doc, doc_id, index=None): + return + + @abstractmethod + def update_many(self, docs=None, doc_ids=None, doc_dict=None, index=None): + return + + @abstractmethod + def update_one(self, doc, doc_id, index=None): + return + + @staticmethod + @abstractmethod + def get_result_size(result): + return + + @abstractmethod + def query_with_scroll(self, dsl, querying_index=None): + return + + @abstractmethod + def query(self, dsl, querying_index=None): + return + + @abstractmethod + def delete_by_query(self, dsl, querying_index=None): + return + + @abstractmethod + def query_pages(self, dsl, querying_index=None): + return + + @abstractmethod + def query_by_id(self, doc_id): + return diff --git a/cumulus_lambda_functions/lib/aws/es_factory.py b/cumulus_lambda_functions/lib/aws/es_factory.py new file mode 100644 index 00000000..f5eb7f82 --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/es_factory.py @@ -0,0 +1,16 @@ +from cumulus_lambda_functions.lib.aws.factory_abstract import FactoryAbstract + + +class ESFactory(FactoryAbstract): + NO_AUTH = 'NO_AUTH' + AWS = 'AWS' + + def get_instance(self, class_type, **kwargs): + ct = class_type.upper() + if ct == self.NO_AUTH: + from cumulus_lambda_functions.lib.aws.es_middleware import ESMiddleware + return ESMiddleware(kwargs['index'], kwargs['base_url'], port=kwargs['port']) + if ct == self.AWS: + from cumulus_lambda_functions.lib.aws.es_middleware_aws import EsMiddlewareAws + return EsMiddlewareAws(kwargs['index'], kwargs['base_url'], port=kwargs['port']) + raise ModuleNotFoundError(f'cannot find ES class for {ct}') diff --git a/cumulus_lambda_functions/lib/aws/es_middleware.py b/cumulus_lambda_functions/lib/aws/es_middleware.py new file mode 100644 index 00000000..d8118d04 --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/es_middleware.py @@ -0,0 +1,220 @@ +import json +import logging + +from elasticsearch import Elasticsearch + +from cumulus_lambda_functions.lib.aws.es_abstract import ESAbstract, DEFAULT_TYPE + +LOGGER = logging.getLogger(__name__) + + +class ESMiddleware(ESAbstract): + + def __init__(self, index, base_url, port=443) -> None: + if any([k is None for k in [index, base_url]]): + raise ValueError(f'index or base_url is None') + self.__index = index + base_url = base_url.replace('https://', '') # hide https + self._engine = Elasticsearch(hosts=[{'host': base_url, 'port': port}]) + + def __validate_index(self, index): + if index is not None: + return index + if self.__index is not None: + return self.__index + raise ValueError('index value is NULL') + + def __get_doc_dict(self, docs=None, doc_ids=None, doc_dict=None): + if doc_dict is None and (docs is None and doc_ids is None): + raise ValueError('must provide either doc dictionary or doc list & id list') + if doc_dict is None: # it comes as a list + if len(docs) != len(doc_ids): + raise ValueError('length of doc and id is different') + doc_dict = {k: v for k, v in zip(doc_ids, docs)} + pass + return doc_dict + + def __check_errors_for_bulk(self, index_result): + if 'errors' not in index_result or index_result['errors'] is False: + return + err_list = [[{'id': v['_id'], 'error': v['error']} for _, v in each.items() if 'error' in v] for each in + index_result['items']] + if len(err_list) < 1: + return + LOGGER.exception('failed to add some items. details: {}'.format(err_list)) + return err_list + + def create_index(self, index_name, index_body): + result = self._engine.indices.create(index=index_name, body=index_body, include_type_name=False) + if 'acknowledged' not in result: + return result + return result['acknowledged'] + + def has_index(self, index_name): + result = self._engine.indices.exists(index=index_name) + return result + + def create_alias(self, index_name, alias_name): + result = self._engine.indices.put_alias(index_name, alias_name) + if 'acknowledged' not in result: + return result + return result['acknowledged'] + + def delete_index(self, index_name): + result = self._engine.indices.delete(index_name) + if 'acknowledged' not in result: + return result + return result['acknowledged'] + + def index_many(self, docs=None, doc_ids=None, doc_dict=None, index=None): + doc_dict = self.__get_doc_dict(docs, doc_ids, doc_dict) + body = [] + for k, v in doc_dict.items(): + body.append({'index': {'_index': index, '_id': k, 'retry_on_conflict': 3}}) + body.append(v) + pass + index = self.__validate_index(index) + try: + index_result = self._engine.bulk(index=index, + body=body, doc_type=DEFAULT_TYPE) + LOGGER.info('indexed. result: {}'.format(index_result)) + return self.__check_errors_for_bulk(index_result) + except: + LOGGER.exception('cannot add indices with ids: {} for index: {}'.format(list(doc_dict.keys()), index)) + return doc_dict + return + + def index_one(self, doc, doc_id, index=None): + index = self.__validate_index(index) + try: + index_result = self._engine.index(index=index, + body=doc, doc_type=DEFAULT_TYPE, id=doc_id) + LOGGER.info('indexed. result: {}'.format(index_result)) + pass + except: + LOGGER.exception('cannot add a new index with id: {} for index: {}'.format(doc_id, index)) + return None + return self + + def update_many(self, docs=None, doc_ids=None, doc_dict=None, index=None): + doc_dict = self.__get_doc_dict(docs, doc_ids, doc_dict) + body = [] + for k, v in doc_dict.items(): + body.append({'update': {'_index': index, '_id': k, 'retry_on_conflict': 3}}) + body.append({'doc': v, 'doc_as_upsert': True}) + pass + index = self.__validate_index(index) + try: + index_result = self._engine.bulk(index=index, + body=body, doc_type=DEFAULT_TYPE) + LOGGER.info('indexed. result: {}'.format(index_result)) + return self.__check_errors_for_bulk(index_result) + except: + LOGGER.exception('cannot update indices with ids: {} for index: {}'.format(list(doc_dict.keys()), + index)) + return doc_dict + return + + def update_one(self, doc, doc_id, index=None): + update_body = { + 'doc': doc, + 'doc_as_upsert': True + } + index = self.__validate_index(index) + try: + update_result = self._engine.update(index=index, + id=doc_id, body=update_body, doc_type=DEFAULT_TYPE) + LOGGER.info('updated. result: {}'.format(update_result)) + pass + except: + LOGGER.exception('cannot update id: {} for index: {}'.format(doc_id, index)) + return None + return self + + @staticmethod + def get_result_size(result): + if isinstance(result['hits']['total'], dict): # fix for different datatype in elastic-search result + return result['hits']['total']['value'] + else: + return result['hits']['total'] + + def query_with_scroll(self, dsl, querying_index=None): + scroll_timeout = '30s' + index = self.__validate_index(querying_index) + dsl['size'] = 10000 # replacing with the maximum size to minimize number of scrolls + params = { + 'index': index, + 'size': 10000, + 'scroll': scroll_timeout, + 'body': dsl, + } + first_batch = self._engine.search(**params) + total_size = self.get_result_size(first_batch) + current_size = len(first_batch['hits']['hits']) + scroll_id = first_batch['_scroll_id'] + while current_size < total_size: # need to scroll + scrolled_result = self._engine.scroll(scroll_id=scroll_id, scroll=scroll_timeout) + scroll_id = scrolled_result['_scroll_id'] + scrolled_result_size = len(scrolled_result['hits']['hits']) + if scrolled_result_size == 0: + break + else: + current_size += scrolled_result_size + first_batch['hits']['hits'].extend(scrolled_result['hits']['hits']) + return first_batch + + def query(self, dsl, querying_index=None): + index = self.__validate_index(querying_index) + return self._engine.search(body=dsl, index=index) + + def delete_by_query(self, dsl, querying_index=None): + index = self.__validate_index(querying_index) + return self._engine.delete_by_query(body=dsl, index=index) + + def __is_querying_next_page(self, targeted_size: int, current_size: int, total_size: int): + if targeted_size < 0: + return current_size > 0 + return current_size > 0 and total_size < targeted_size + + def query_pages(self, dsl, querying_index=None): + if 'sort' not in dsl: + raise ValueError('missing `sort` in DSL. Make sure sorting is unique') + index = self.__validate_index(querying_index) + targeted_size = dsl['sort'] if 'size' in dsl else -1 + dsl['size'] = 10000 # replacing with the maximum size to minimize number of scrolls + params = { + 'index': index, + 'size': 10000, + 'body': dsl, + } + LOGGER.debug(f'dsl: {dsl}') + result_list = [] + total_size = 0 + result_batch = self._engine.search(**params) + result_list.extend(result_batch['hits']['hits']) + current_size = len(result_batch['hits']['hits']) + total_size += current_size + while self.__is_querying_next_page(targeted_size, current_size, total_size): + params['body']['search_after'] = result_batch['hits']['hits'][-1]['sort'] + result_batch = self._engine.search(**params) + result_list.extend(result_batch['hits']['hits']) + current_size = len(result_batch['hits']['hits']) + total_size += current_size + return { + 'hits': { + 'hits': result_list, + 'total': total_size, + } + } + + def query_by_id(self, doc_id): + index = self.__validate_index(None) + dsl = { + 'query': { + 'term': {'_id': doc_id} + } + } + result = self._engine.search(index=index, body=dsl) + if self.get_result_size(result) < 1: + return None + return result['hits']['hits'][0] diff --git a/cumulus_lambda_functions/lib/aws/es_middleware_aws.py b/cumulus_lambda_functions/lib/aws/es_middleware_aws.py new file mode 100644 index 00000000..9e2701cd --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/es_middleware_aws.py @@ -0,0 +1,29 @@ +import logging + +from requests_aws4auth import AWS4Auth + +from cumulus_lambda_functions.lib.aws.aws_cred import AwsCred +from cumulus_lambda_functions.lib.aws.es_middleware import ESMiddleware +from elasticsearch import Elasticsearch, RequestsHttpConnection + +LOGGER = logging.getLogger(__name__) + + +class EsMiddlewareAws(ESMiddleware): + + def __init__(self, index, base_url, port=443) -> None: + super().__init__(index, base_url, port) + base_url = base_url.replace('https://', '') # hide https + self._index = index + aws_cred = AwsCred() + service = 'es' + credentials = aws_cred.get_session().get_credentials() + aws_auth = AWS4Auth(credentials.access_key, credentials.secret_key, aws_cred.region, service, + session_token=credentials.token) + self._engine = Elasticsearch( + hosts=[{'host': base_url, 'port': port}], + http_auth=aws_auth, + use_ssl=True, + verify_certs=True, + connection_class=RequestsHttpConnection + ) diff --git a/cumulus_lambda_functions/lib/aws/factory_abstract.py b/cumulus_lambda_functions/lib/aws/factory_abstract.py new file mode 100644 index 00000000..16de81cf --- /dev/null +++ b/cumulus_lambda_functions/lib/aws/factory_abstract.py @@ -0,0 +1,7 @@ +from abc import ABC, abstractmethod + + +class FactoryAbstract(ABC): + @abstractmethod + def get_instance(self, class_type, **kwargs): + return diff --git a/cumulus_lambda_functions/lib/uds_db/__init__.py b/cumulus_lambda_functions/lib/uds_db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cumulus_lambda_functions/lib/uds_db/db_constants.py b/cumulus_lambda_functions/lib/uds_db/db_constants.py new file mode 100644 index 00000000..52e1a08e --- /dev/null +++ b/cumulus_lambda_functions/lib/uds_db/db_constants.py @@ -0,0 +1,7 @@ +class DBConstants: + authorization_index = 'authorization_mappings' + authorized_group_name_key = 'cognito_group' + action_key = 'action' + resource_key = 'resource' + tenant_venue = 'tenant_venue' + tenant = 'tenant' diff --git a/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py b/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py index c7d216e8..f4a1093d 100644 --- a/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py +++ b/cumulus_lambda_functions/lib/utils/lambda_api_gateway_utils.py @@ -1,3 +1,5 @@ +import base64 +import json from copy import deepcopy from cumulus_lambda_functions.lib.json_validator import JsonValidator @@ -8,6 +10,7 @@ 'required': [ 'requestContext', 'headers', + 'httpMethod', ], 'properties': { 'requestContext': { @@ -25,18 +28,116 @@ 'type': 'object', 'required': [ 'Host', + 'Authorization', ], 'properties': { 'Host': { 'type': 'string' + }, + 'Authorization': { + 'type': 'string' } } - } + }, + 'httpMethod': {'type': 'string'}, } } LOGGER = LambdaLoggerGenerator.get_logger(__name__, LambdaLoggerGenerator.get_level_from_env()) +SAMPLE = { + "resource": "/am-uds-dapa/collections/{collectionId}", + "path": "/am-uds-dapa/collections/CUMULUS_DAPA_UNIT_TEST___1665800977", + "httpMethod": "GET", + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Authorization": "Bearer eyJraWQiOiJzdE42WWl0eGxWZmJnY1ByRnJLWVQ1MEdjVWRIZWNBaWFKQ09peUxLVHNZPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiIwMGRkYTdmNy1mNjE4LTRmNDMtYWFmNC1iYmM4YmExNDc2ODAiLCJjb2duaXRvOmdyb3VwcyI6WyJVbml0eV9WaWV3ZXIiLCJUZXN0X0dyb3VwIiwiVW5pdHlfQWRtaW4iXSwiaXNzIjoiaHR0cHM6XC9cL2NvZ25pdG8taWRwLnVzLXdlc3QtMi5hbWF6b25hd3MuY29tXC91cy13ZXN0LTJfRkxEeVhFMm1PIiwiY2xpZW50X2lkIjoiN2ExZmdsbTJkNTRlb2dnajEzbGNjaXZwMjUiLCJvcmlnaW5fanRpIjoiNzc3YmZkNDEtZTY1OS00M2Y0LThmYzEtNGFkOTEwNDE1NDdiIiwiZXZlbnRfaWQiOiI5YjM3OGE3ZS03OWQxLTQyMmYtODEwZi1lM2Q2Mjk2ODNiZDEiLCJ0b2tlbl91c2UiOiJhY2Nlc3MiLCJzY29wZSI6ImF3cy5jb2duaXRvLnNpZ25pbi51c2VyLmFkbWluIiwiYXV0aF90aW1lIjoxNjY1Nzc1Nzc3LCJleHAiOjE2NjU3NzkzNzcsImlhdCI6MTY2NTc3NTc3NywianRpIjoiMDk5OTA3YTMtMWVhNS00MmRiLTg4ZmMtNGU5MjQzM2FhOTA0IiwidXNlcm5hbWUiOiJ3cGh5byJ9.CiFHUKqz3q3TwW6XOPH2lkglfWO1LRk-ly-mB280GoFyGBhzSjnWHbo_U-NFmI7VsilywLMXFkif2IQ7AJSt9Cj2pja5ohrmDOFfQR_EuSSo-skSYElVaMYmIkRfVWfVa6gyByOGho5utANI0a6y9nnazvdp3ebXpdVUrNHC3yLns9-CijW-2jEvNDvEoaUZxQp06H29mcb4Iupc_SFYaVzNt3Xf_eumbyV2c0tdBEy-aRsNSOwhLXEREVJyGjHMsaSi8q2FUpHigj9ORfGtntjqESiCLCDc7wAhx6eqEZ8bfC87ck33UiFhYsamaNFHXbYTLl-uN8W5yyi8TAvJDA", + "Host": "k3a3qmarxh.execute-api.us-west-2.amazonaws.com", + "User-Agent": "python-requests/2.27.1", + "X-Amzn-Trace-Id": "Root=1-6349b8e1-26c2c9e702a5cd546ba96c81", + "X-Forwarded-For": "137.79.228.173", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "multiValueHeaders": { + "Accept": [ + "*/*" + ], + "Accept-Encoding": [ + "gzip, deflate" + ], + "Authorization": [ + "Bearer eyJraWQiOiJzdE42WWl0eGxWZmJnY1ByRnJLWVQ1MEdjVWRIZWNBaWFKQ09peUxLVHNZPSIsImFsZyI6IlJTMjU2In0.eyJzdWIiOiIwMGRkYTdmNy1mNjE4LTRmNDMtYWFmNC1iYmM4YmExNDc2ODAiLCJjb2duaXRvOmdyb3VwcyI6WyJVbml0eV9WaWV3ZXIiLCJUZXN0X0dyb3VwIiwiVW5pdHlfQWRtaW4iXSwiaXNzIjoiaHR0cHM6XC9cL2NvZ25pdG8taWRwLnVzLXdlc3QtMi5hbWF6b25hd3MuY29tXC91cy13ZXN0LTJfRkxEeVhFMm1PIiwiY2xpZW50X2lkIjoiN2ExZmdsbTJkNTRlb2dnajEzbGNjaXZwMjUiLCJvcmlnaW5fanRpIjoiNzc3YmZkNDEtZTY1OS00M2Y0LThmYzEtNGFkOTEwNDE1NDdiIiwiZXZlbnRfaWQiOiI5YjM3OGE3ZS03OWQxLTQyMmYtODEwZi1lM2Q2Mjk2ODNiZDEiLCJ0b2tlbl91c2UiOiJhY2Nlc3MiLCJzY29wZSI6ImF3cy5jb2duaXRvLnNpZ25pbi51c2VyLmFkbWluIiwiYXV0aF90aW1lIjoxNjY1Nzc1Nzc3LCJleHAiOjE2NjU3NzkzNzcsImlhdCI6MTY2NTc3NTc3NywianRpIjoiMDk5OTA3YTMtMWVhNS00MmRiLTg4ZmMtNGU5MjQzM2FhOTA0IiwidXNlcm5hbWUiOiJ3cGh5byJ9.CiFHUKqz3q3TwW6XOPH2lkglfWO1LRk-ly-mB280GoFyGBhzSjnWHbo_U-NFmI7VsilywLMXFkif2IQ7AJSt9Cj2pja5ohrmDOFfQR_EuSSo-skSYElVaMYmIkRfVWfVa6gyByOGho5utANI0a6y9nnazvdp3ebXpdVUrNHC3yLns9-CijW-2jEvNDvEoaUZxQp06H29mcb4Iupc_SFYaVzNt3Xf_eumbyV2c0tdBEy-aRsNSOwhLXEREVJyGjHMsaSi8q2FUpHigj9ORfGtntjqESiCLCDc7wAhx6eqEZ8bfC87ck33UiFhYsamaNFHXbYTLl-uN8W5yyi8TAvJDA" + ], + "Host": [ + "k3a3qmarxh.execute-api.us-west-2.amazonaws.com" + ], + "User-Agent": [ + "python-requests/2.27.1" + ], + "X-Amzn-Trace-Id": [ + "Root=1-6349b8e1-26c2c9e702a5cd546ba96c81" + ], + "X-Forwarded-For": [ + "137.79.228.173" + ], + "X-Forwarded-Port": [ + "443" + ], + "X-Forwarded-Proto": [ + "https" + ] + }, + "queryStringParameters": None, + "multiValueQueryStringParameters": None, + "pathParameters": { + "collectionId": "CUMULUS_DAPA_UNIT_TEST___1665800977" + }, + "stageVariables": { + "VPCLINK": "czcxgk" + }, + "requestContext": { + "resourceId": "cv4985", + "authorizer": { + "numberKey": "123", + "booleanKey": "true", + "stringKey": "stringval", + "principalId": "user", + "integrationLatency": 303 + }, + "resourcePath": "/am-uds-dapa/collections/{collectionId}", + "httpMethod": "GET", + "extendedRequestId": "aAnTUHrSPHcFcGg=", + "requestTime": "14/Oct/2022:19:30:41 +0000", + "path": "/dev/am-uds-dapa/collections/CUMULUS_DAPA_UNIT_TEST___1665800977", + "accountId": "884500545225", + "protocol": "HTTP/1.1", + "stage": "dev", + "domainPrefix": "k3a3qmarxh", + "requestTimeEpoch": 1665775841896, + "requestId": "67b76cda-e49d-449e-ad3d-8ba6d42a4ce4", + "identity": { + "cognitoIdentityPoolId": None, + "accountId": None, + "cognitoIdentityId": None, + "caller": None, + "sourceIp": "137.79.228.173", + "principalOrgId": None, + "accessKey": None, + "cognitoAuthenticationType": None, + "cognitoAuthenticationProvider": None, + "userArn": None, + "userAgent": "python-requests/2.27.1", + "user": None + }, + "domainName": "k3a3qmarxh.execute-api.us-west-2.amazonaws.com", + "apiId": "k3a3qmarxh" + }, + "body": None, + "isBase64Encoded": False +} + class LambdaApiGatewayUtils: def __init__(self, event: dict, default_limit: int = 10): self.__event = event @@ -45,6 +146,23 @@ def __init__(self, event: dict, default_limit: int = 10): if api_gateway_event_validator_result is not None: raise ValueError(f'invalid event: {api_gateway_event_validator_result}. event: {event}') + def get_authorization_info(self): + """ + :return: + """ + action = self.__event['httpMethod'] + resource = self.__event['requestContext']['path'] + bearer_token = self.__event['headers']['Authorization'] + username_part = bearer_token.split('.')[1] + jwt_decoded = base64.standard_b64decode(username_part.encode()).decode() + jwt_decoded = json.loads(jwt_decoded) + username = jwt_decoded['username'] + return { + 'username': username, + 'action': action, + 'resource': resource, + } + def __get_current_page(self): try: requesting_base_url = f"https://{self.__event['headers']['Host']}{self.__event['requestContext']['path']}" diff --git a/etc/elasticsearch_index/alias_pointer.json b/etc/elasticsearch_index/alias_pointer.json new file mode 100644 index 00000000..3b59e720 --- /dev/null +++ b/etc/elasticsearch_index/alias_pointer.json @@ -0,0 +1,5 @@ +{ + "actions" : [ + {"add" : {"index" : "authorization_mappings_v1", "alias" : "authorization_mappings"}} + ] +} \ No newline at end of file diff --git a/etc/elasticsearch_index/authorization_mappings_v1.json b/etc/elasticsearch_index/authorization_mappings_v1.json new file mode 100644 index 00000000..5e286f5c --- /dev/null +++ b/etc/elasticsearch_index/authorization_mappings_v1.json @@ -0,0 +1,15 @@ +{ + "settings" : { + "number_of_shards" : 3, + "number_of_replicas" : 2 + }, + "mappings": { + "properties": { + "action": {"type": "keyword"}, + "resource": {"type": "keyword"}, + "cognito_group": {"type": "keyword"}, + "tenant": {"type": "keyword"}, + "tenant_venue": {"type": "keyword"} + } + } +} \ No newline at end of file diff --git a/etc/elasticsearch_index/setup.txt b/etc/elasticsearch_index/setup.txt new file mode 100644 index 00000000..1e2a6d53 --- /dev/null +++ b/etc/elasticsearch_index/setup.txt @@ -0,0 +1,9 @@ +export ES_HOST=localhost:9201 + +curl -X PUT "$ES_HOST/authorization_mappings_v1" --data "@authorization_mappings_v1.json" -H 'Content-Type:application/json' + +curl -X POST "$ES_HOST/_aliases" --data "@alias_pointer.json" -H 'Content-Type:application/json' + + +deleting indices +curl -X DELETE "$ES_HOST/authorization_mappings_v1" \ No newline at end of file diff --git a/setup.py b/setup.py index cbd51866..f17f3ecc 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,9 @@ 'pystac', 'jsonschema', 'fastjsonschema', 'xmltodict', - 'requests' + 'requests===2.27.1', + 'elasticsearch===7.13.4', + 'requests_aws4auth', ] flask_requires = [ diff --git a/tests/cumulus_lambda_functions/lib/authorization/__init__.py b/tests/cumulus_lambda_functions/lib/authorization/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cumulus_lambda_functions/lib/authorization/test_uds_authorizer_es_identity_pool.py b/tests/cumulus_lambda_functions/lib/authorization/test_uds_authorizer_es_identity_pool.py new file mode 100644 index 00000000..dc0de857 --- /dev/null +++ b/tests/cumulus_lambda_functions/lib/authorization/test_uds_authorizer_es_identity_pool.py @@ -0,0 +1,55 @@ +import os +from time import sleep +from unittest import TestCase + +from cumulus_lambda_functions.lib.authorization.uds_authorizer_es_identity_pool import UDSAuthorizorEsIdentityPool + + +class TestUDSAuthorizorEsIdentityPool(TestCase): + def test_01(self): + os.environ['ES_URL'] = 'https://search-uds-es-test-2-olhpweojwudrginxdizzn3itt4.us-west-2.es.amazonaws.com' + os.environ['AUTHORIZATION_INDEX'] = 'authorization_mappings' + + authorizer = UDSAuthorizorEsIdentityPool('us-west-2_FLDyXE2mO') + + authorizer.add_authorized_group(['PUT', 'POST'], ['COLLECTION'], 'unitty_project_1', 'DEV_001', 'sample_group_1A') + authorizer.add_authorized_group(['PUT', 'POST'], ['COLLECTION', 'GET'], 'unitty_project_2', 'DEV_001', 'sample_group_2A') + authorizer.add_authorized_group(['GET'], ['COLLECTION', 'GET'], 'unitty_project_1', 'DEV_001', 'sample_group_1B') + authorizer.add_authorized_group(['DELETE'], ['COLLECTION'], 'unitty_project_1', 'DEV_001', 'sample_group_1C') + sleep(2) + authorized_groups = authorizer.list_authorized_groups_for('unitty_project_1', 'DEV_001') + self.assertEqual(len(authorized_groups), 3, f'invalid length of result') + authorizer.delete_authorized_group('unitty_project_1', 'DEV_001', 'sample_group_1A') + authorizer.delete_authorized_group('unitty_project_1', 'DEV_001', 'sample_group_1B') + authorizer.delete_authorized_group('unitty_project_1', 'DEV_001', 'sample_group_1C') + authorizer.delete_authorized_group('unitty_project_2', 'DEV_001', 'sample_group_2A') + sleep(2) + authorized_groups = authorizer.list_authorized_groups_for('unitty_project_1', 'DEV_001') + self.assertEqual(len(authorized_groups), 0, f'invalid length of result') + authorizer.update_authorized_group(['PUT', 'POST'], ['COLLECTION'], 'unitty_project_1', 'DEV_001', 'sample_group_1A') + sleep(2) + authorized_groups = authorizer.list_authorized_groups_for('unitty_project_1', 'DEV_001') + self.assertEqual(len(authorized_groups), 1, f'invalid length of result') + return + + def test_02(self): + os.environ['ES_URL'] = 'https://search-uds-es-test-2-olhpweojwudrginxdizzn3itt4.us-west-2.es.amazonaws.com' + os.environ['AUTHORIZATION_INDEX'] = 'authorization_mappings' + + authorizer = UDSAuthorizorEsIdentityPool('us-west-2_FLDyXE2mO') + authorizer.add_authorized_group(['PUT', 'POST'], ['COLLECTION'], 'unitty_project_1', 'DEV_001', 'Unity_Viewer') + authorizer.add_authorized_group(['PUT'], ['COLLECTION', 'GRANULE'], 'unitty_project_2', 'DEV_001', 'Test_Group') + sleep(2) + self.assertEqual(len(authorizer.get_authorized_tenant('wphyo', 'GET', 'COLLECTION')), 0, f'wrong length of result') + self.assertEqual(len(authorizer.get_authorized_tenant('wphyo', 'PUT', 'COLLECTION')), 2, f'wrong length of result') + self.assertEqual(len(authorizer.get_authorized_tenant('wphyo', 'GET', 'GRANULE')), 0, f'wrong length of result') + put_authorized_list = authorizer.get_authorized_tenant('wphyo', 'PUT', 'GRANULE') + self.assertEqual(len(put_authorized_list), 1, f'wrong length of result') + self.assertEqual(put_authorized_list[0]['tenant'], 'unitty_project_2', 'wrong project for PUT GRANULES') + self.assertEqual(put_authorized_list[0]['tenant_venue'], 'DEV_001', 'wrong venue for PUT GRANULES') + self.assertEqual(len(authorizer.get_authorized_tenant('wphyo', 'POST', 'COLLECTION')), 1, f'wrong length of result') + self.assertEqual(len(authorizer.get_authorized_tenant('wphyo', 'DELETE', 'COLLECTION')), 0, f'wrong length of result') + authorizer.delete_authorized_group('unitty_project_1', 'DEV_001', 'Unity_Viewer') + authorizer.delete_authorized_group('unitty_project_2', 'DEV_001', 'Test_Group') + + return diff --git a/tests/cumulus_lambda_functions/lib/aws/test_aws_cognito.py b/tests/cumulus_lambda_functions/lib/aws/test_aws_cognito.py new file mode 100644 index 00000000..9afc20a6 --- /dev/null +++ b/tests/cumulus_lambda_functions/lib/aws/test_aws_cognito.py @@ -0,0 +1,13 @@ +from unittest import TestCase + +from cumulus_lambda_functions.lib.aws.aws_cognito import AwsCognito + + +class TestAwsCognitor(TestCase): + def test_01(self): + cognito = AwsCognito('us-west-2_FLDyXE2mO') + wphyo_groups = cognito.get_groups('wphyo') + self.assertTrue(isinstance(wphyo_groups, list), f'response is not list. {wphyo_groups}') + self.assertTrue(len(wphyo_groups) > 0, f'empty list') + print(wphyo_groups) + return diff --git a/tf-module/unity-cumulus/es_access_policy.json b/tf-module/unity-cumulus/es_access_policy.json new file mode 100644 index 00000000..fab4ce6e --- /dev/null +++ b/tf-module/unity-cumulus/es_access_policy.json @@ -0,0 +1,11 @@ + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "es:*", + "Principal": "*", + "Effect": "Allow", + "Resource": "${es_resource}" + } + ] + } \ No newline at end of file diff --git a/tf-module/unity-cumulus/opensearch.tf b/tf-module/unity-cumulus/opensearch.tf new file mode 100644 index 00000000..b4797fb0 --- /dev/null +++ b/tf-module/unity-cumulus/opensearch.tf @@ -0,0 +1,46 @@ +resource "aws_elasticsearch_domain" "uds-es" { + domain_name = "${var.prefix}-es" +// engine_version = "Elasticsearch_7.10" +// Not supported that in elastic_search + elasticsearch_version = "7.10" + cluster_config { + instance_count = var.uds_es_cluster_instance_count + instance_type = var.uds_es_cluster_instance_type + } + +// advanced_security_options { +// enabled = false +// anonymous_auth_enabled = true +// internal_user_database_enabled = true +// master_user_options { +// master_user_name = "example" +// master_user_password = "Barbarbarbar1!" +// } +// } + + vpc_options { + subnet_ids = var.cumulus_lambda_subnet_ids + security_group_ids = local.security_group_ids_set ? var.security_group_ids : [aws_security_group.unity_cumulus_lambda_sg[0].id] + } + encrypt_at_rest { + enabled = true + } + node_to_node_encryption { + enabled = true + } + ebs_options { + ebs_enabled = true + volume_type = "gp2" +// throughput = 125 + volume_size = 30 + } + access_policies = templatefile( + "${path.module}/es_access_policy.json", + { + es_resource: "arn:aws:es:${var.aws_region}:${var.account_id}:domain/${var.prefix}-es/*" + } + ) + tags = { +// Domain = "TestDomain" + } +} \ No newline at end of file diff --git a/tf-module/unity-cumulus/variables.tf b/tf-module/unity-cumulus/variables.tf index ee10f42f..b5d0fb2e 100644 --- a/tf-module/unity-cumulus/variables.tf +++ b/tf-module/unity-cumulus/variables.tf @@ -3,6 +3,10 @@ variable "log_level" { default = "20" description = "Lambda Log Level. Follow Python3 log level numbers info=20, warning=30, etc..." } +variable "account_id" { + type = string + description = "AWS Account ID" +} variable "prefix" { type = string } @@ -52,3 +56,15 @@ variable "cumulus_base" { variable "lambda_processing_role_arn" { type = string } + +variable "uds_es_cluster_instance_count" { + type = number + default = 2 + description = "How many EC2 instances for Opensearch" +} + +variable "uds_es_cluster_instance_type" { + type = string + default = "r5.large.search" + description = "EC2 instance type for Opensearch" +} \ No newline at end of file