Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

breaking: allow custom-metadata in granules dapa query #254

Merged
merged 48 commits into from
Jan 10, 2024
Merged
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
b961d56
feat: allow docs and redoc pages
wphyojpl Oct 9, 2023
7360585
fix: custom open url
wphyojpl Oct 9, 2023
3df0702
fix: get docs page to work
wphyojpl Oct 9, 2023
07e179d
Merge branch 'develop' of github.com:unity-sds/unity-data-services in…
wphyojpl Oct 9, 2023
70ca1d9
feat: return openapi via endpoint
wphyojpl Oct 9, 2023
64165e3
chore: updating documentation
wphyojpl Oct 13, 2023
827157f
fix: wrong schema
wphyojpl Oct 13, 2023
bd8e2c8
fix: adding some documentation
wphyojpl Oct 17, 2023
2fce468
feat: cql parser
wphyojpl Oct 18, 2023
9eb8fbf
fix: start to convert json to text
wphyojpl Oct 19, 2023
5550488
chore: consolidate granules custom data query into a single class
wphyojpl Oct 24, 2023
dbbc6e9
fix: update cql for query
wphyojpl Nov 7, 2023
118e525
chore: merge from develop
wphyojpl Nov 9, 2023
4142ec4
fix: update custom metadata update
wphyojpl Nov 9, 2023
8809755
chore: prepping granules to es work
wphyojpl Nov 9, 2023
07d2b8c
feat: more prep work
wphyojpl Nov 13, 2023
a7d2dc5
fix: raw test to see if new code works so far
wphyojpl Nov 13, 2023
b91dfcb
fix: add checksum & sum to granule item transformer
wphyojpl Nov 14, 2023
69cd9b9
granules index is in stac form now
wphyojpl Nov 16, 2023
a65fe48
feat: store stac document in ES
wphyojpl Nov 16, 2023
a22b105
fix: allow partial cumulus to stac
wphyojpl Nov 16, 2023
b6ad8f7
feat: update terraform
wphyojpl Nov 16, 2023
4b9eb70
fix: no filter policy at this moment
wphyojpl Nov 16, 2023
9f26c70
fix: missing argument when calling a method
wphyojpl Nov 16, 2023
35c7194
fix: missing comma
wphyojpl Nov 16, 2023
4d297cc
fix: update retrieve custom metadata for stac mapping
wphyojpl Nov 17, 2023
87cd865
fix: need extra properties
wphyojpl Nov 17, 2023
b80c76d
feat: add pagination + using ES
wphyojpl Nov 22, 2023
96f72b4
fix: still fixing properties
wphyojpl Nov 27, 2023
724e6bf
fix: trying properties again
wphyojpl Nov 27, 2023
53bff61
fix: offset is not int anymore
wphyojpl Nov 27, 2023
ae9b0c5
fix: validated logic for pagination
wphyojpl Nov 27, 2023
2b32ac5
fix: typo
wphyojpl Nov 27, 2023
5f8aad1
fix: some tweaks
wphyojpl Nov 27, 2023
0dc9ce0
fix: need geometry as point
wphyojpl Nov 27, 2023
15753fc
chore: dummy commit
wphyojpl Nov 27, 2023
aa48858
chore: dummy commit
wphyojpl Nov 27, 2023
fea9ce6
fix: need snake case event time + not returning class
wphyojpl Nov 27, 2023
eea0efa
fix: some minor tweaks
wphyojpl Dec 1, 2023
d416c6f
fix: extra format
wphyojpl Dec 4, 2023
da6a46d
fix: testing queries with filter cql
wphyojpl Dec 4, 2023
4ba2669
fix: just normal query. not query by page
wphyojpl Dec 4, 2023
3ddcab5
chore: add test case on pagination
wphyojpl Dec 4, 2023
ba31579
fix: api gateway terraform
wphyojpl Jan 2, 2024
58f2a7b
chore: consolidate same config variable
wphyojpl Jan 2, 2024
224c936
fix: add open api endpoint without authorization
wphyojpl Jan 2, 2024
0a519ea
Merge branch 'develop' into dapa-custom-search
wphyojpl Jan 3, 2024
6fb9a69
Merge remote-tracking branch 'origin/develop' into dapa-custom-search
ngachung Jan 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions cumulus_lambda_functions/lib/cql_parser.py
Original file line number Diff line number Diff line change
@@ -18,17 +18,21 @@ def __valide_core_expression(self, parsed_obj):
def __transform_this(self, parsed_obj):
if isinstance(parsed_obj, And):
return {
'must': [
self.__transform_this(parsed_obj.lhs),
self.__transform_this(parsed_obj.rhs),
]
'bool': {
'must': [
self.__transform_this(parsed_obj.lhs),
self.__transform_this(parsed_obj.rhs),
]
}
}
if isinstance(parsed_obj, Or):
return {
'should': [
self.__transform_this(parsed_obj.lhs),
self.__transform_this(parsed_obj.rhs),
]
'bool': {
'should': [
self.__transform_this(parsed_obj.lhs),
self.__transform_this(parsed_obj.rhs),
]
}
}
if isinstance(parsed_obj, IsNull):
return {
40 changes: 40 additions & 0 deletions cumulus_lambda_functions/uds_api/dapa/granules_dapa_query.py
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
import os

from cumulus_lambda_functions.cumulus_stac.item_transformer import ItemTransformer
from cumulus_lambda_functions.lib.cql_parser import CqlParser

from cumulus_lambda_functions.lib.uds_db.uds_collections import UdsCollections

@@ -18,6 +19,9 @@
class GranulesDapaQuery:
def __init__(self, collection_id, limit, offset, datetime, filter_input, pagination_links):
self.__pagination_links = pagination_links
self.__limit = limit
self.__offset = offset
self.__filter_input = filter_input
page_number = (offset // limit) + 1
if 'CUMULUS_LAMBDA_PREFIX' not in os.environ:
raise EnvironmentError('missing key: CUMULUS_LAMBDA_PREFIX')
@@ -30,6 +34,37 @@ def __init__(self, collection_id, limit, offset, datetime, filter_input, paginat
self.__collection_id = collection_id
self.__get_time_range(datetime)
self.__get_filter(filter_input)
self.__es_granules_result = None # this is where Elasticsearch granules result is stored

def __custom_metadata_query(self):
if self.__filter_input is None:
return self
LOGGER.debug(f'filter_input: {self.__filter_input}')
dsl_query = CqlParser().transform(self.__filter_input)
LOGGER.debug(f'CqlParser result: {dsl_query}')
custom_metadata_query_dsl = {
'from': self.__offset,
'size': self.__limit,
'query': {
'bool': {
'must': [

dsl_query,
]
}
}
}

LOGGER.debug(f'custom_metadata_query_dsl: {custom_metadata_query_dsl}')
collection_identifier = UdsCollections.decode_identifier(self.__collection_id)
LOGGER.debug(f'custom_metadata_query_dsl: {custom_metadata_query_dsl}')
custom_metadata_result = GranulesDbIndex().dsl_search(collection_identifier.tenant, collection_identifier.venue,
custom_metadata_query_dsl)
LOGGER.debug(f'custom_metadata_result: {custom_metadata_result}')
custom_metadata_result = [k['_source'] for k in custom_metadata_result['hits']['hits']]
self.__es_granules_result = {k['granule_id']: k for k in custom_metadata_result}
return self


def __get_time_range(self, datetime: str):
if datetime is None:
@@ -151,6 +186,11 @@ def __get_custom_metadata(self, cumulus_result) -> dict:

def start(self):
try:
self.__custom_metadata_query()
if self.__es_granules_result is not None:
# already queried custom metadata.
# just need to find those granule ids from Cumulus.
self.__get_filter('granules_id', [k for k in self.__es_granules_result.keys()])
cumulus_result = self.__cumulus.query_direct_to_private_api(self.__cumulus_lambda_prefix, False)
if 'server_error' in cumulus_result:
return {
11 changes: 11 additions & 0 deletions tests/cumulus_lambda_functions/lib/test_cql_parser.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import json
from unittest import TestCase

from cumulus_lambda_functions.lib.cql_parser import CqlParser
@@ -12,3 +13,13 @@ def test_01(self):
expected_result = {'terms': {'id': ['G1', 'G2', 'G3']}}
self.assertEqual(expected_result, parsed_result)
return

def test_02(self):
granule_ids = ['G1', 'G2', 'G3']
granule_ids = [f"'{k}'" for k in granule_ids]
filter_cql = f"id in ({','.join(granule_ids)}) and tags = 'level-3' and (time1 < 34 or time1 > 14)"
parsed_result = CqlParser().transform(filter_cql)
print(parsed_result)
expected_result = {'bool': {'must': [{'bool': {'must': [{'terms': {'id': ['G1', 'G2', 'G3']}}, {'term': {'tags': 'level-3'}}]}}, {'bool': {'should': [{'range': {'time1': {'lt': 34}}}, {'range': {'time1': {'gt': 14}}}]}}]}}
self.assertEqual(sorted(json.dumps(expected_result)), sorted(json.dumps(parsed_result)))
return
16 changes: 13 additions & 3 deletions tests/integration_tests/test_custom_metadata_end_to_end.py
Original file line number Diff line number Diff line change
@@ -39,9 +39,9 @@ def setUp(self) -> None:
base64.standard_b64decode(os.environ.get('PASSWORD')).decode())
self._url_prefix = f'{os.environ.get("UNITY_URL")}/{os.environ.get("UNITY_STAGE", "sbx-uds-dapa")}'
self.tenant = 'uds_sandbox' # 'uds_local_test' # 'uds_sandbox'
self.tenant_venue = 'dev' # 'DEV1' # 'dev'
self.tenant_venue = 'DEV1' # 'DEV1' # 'dev'
self.collection_name = 'uds_collection' # 'uds_collection' # 'sbx_collection'
self.collection_version = '2310231744' # '2309141300'
self.collection_version = '2310241151' # '2309141300'
self.custom_metadata_body = {
'tag': {'type': 'keyword'},
'c_data1': {'type': 'long'},
@@ -127,6 +127,16 @@ def test_03_create_collection(self):
)
self.assertEqual(query_result.status_code, 202, f'wrong status code. {query_result.text}')
sleep(60)
post_url = post_url if post_url.endswith('/') else f'{post_url}/'
collection_created_result = requests.get(url=f'{post_url}{temp_collection_id}', headers=headers)
self.assertEqual(collection_created_result.status_code, 200,
f'wrong status code. {collection_created_result.text}')
collection_created_result = json.loads(collection_created_result.text)
self.assertTrue('features' in collection_created_result,
f'features not in collection_created_result: {collection_created_result}')
self.assertEqual(len(collection_created_result['features']), 1, f'wrong length: {collection_created_result}')
self.assertEqual(collection_created_result['features'][0]['id'], temp_collection_id, f'wrong id')
print(collection_created_result)
return

def test_04_upload_sample_granule(self):
@@ -254,7 +264,7 @@ def test_04_upload_sample_granule(self):
<val>8c3ae101-8f7c-46c8-b5c6-63e7b6d3c8cd</val>
</keyval>
</cas:metadata>''')
stac_item = Item(id='test_file01',
stac_item = Item(id=f'{temp_collection_id}:test_file01',
geometry={
"type": "Point",
"coordinates": [0.0, 0.0]