Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Environment seperation cleanup, subscription API #53

Merged
merged 7 commits into from
Mar 27, 2023
Merged

Environment seperation cleanup, subscription API #53

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
436ef6b
Env seperation cleanup, subscription API
dogversioning Mar 24, 2023
8edead9
print trim
dogversioning Mar 27, 2023
3f53371
trimming debug
dogversioning Mar 27, 2023
70ea97d
mocker linting, 3.9 cleanup, resorted athena permissions
dogversioning Mar 27, 2023
9e7d542
manual lint run and correction
dogversioning Mar 27, 2023
4e06b04
documentation branch leak plugging
dogversioning Mar 27, 2023
cff7d4e
black targeting
dogversioning Mar 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ repos:
# supported by your project here, or alternatively use
# pre-commit's default_language_version, see
# https://pre-commit.com/#top_level-default_language_version
language_version: python3.11
language_version: python3.10
mikix marked this conversation as resolved.
Show resolved Hide resolved
1 change: 1 addition & 0 deletions .python-version
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
aggregator-env-3.10.10
mikix marked this conversation as resolved.
Show resolved Hide resolved
141 changes: 0 additions & 141 deletions events/event-dashboard-chart-request.json
View file
Open in desktop

This file was deleted.

125 changes: 125 additions & 0 deletions events/event-get-chart-data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
{
"resource": "/chart_data/{subscription_name}",
"path": "/chart_data/covid__encounter",
"httpMethod": "GET",
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate, br",
"CloudFront-Forwarded-Proto": "https",
"CloudFront-Is-Desktop-Viewer": "true",
"CloudFront-Is-Mobile-Viewer": "false",
"CloudFront-Is-SmartTV-Viewer": "false",
"CloudFront-Is-Tablet-Viewer": "false",
"CloudFront-Viewer-ASN": "40127",
"CloudFront-Viewer-Country": "US",
"Host": "effmuaxft2.execute-api.us-east-1.amazonaws.com",
"Postman-Token": "10f731c3-bb8d-4720-971c-7fd5e7cf0065",
"User-Agent": "PostmanRuntime/7.31.3",
"Via": "1.1 3ae9464b3a12f9a00e97e3c81ee98466.cloudfront.net (CloudFront)",
"X-Amz-Cf-Id": "JWyqc8QN24nqpUdGuej_gc35vpExxV017XJD8EXxM8x1OupUzCGYbg==",
"X-Amzn-Trace-Id": "Root=1-641c7f2c-5c612f3e389094297688efa0",
"X-Forwarded-For": "134.174.21.156, 70.132.21.75",
"X-Forwarded-Port": "443",
"X-Forwarded-Proto": "https"
},
"multiValueHeaders": {
"Accept": [
"*/*"
],
"Accept-Encoding": [
"gzip, deflate, br"
],
"CloudFront-Forwarded-Proto": [
"https"
],
"CloudFront-Is-Desktop-Viewer": [
"true"
],
"CloudFront-Is-Mobile-Viewer": [
"false"
],
"CloudFront-Is-SmartTV-Viewer": [
"false"
],
"CloudFront-Is-Tablet-Viewer": [
"false"
],
"CloudFront-Viewer-ASN": [
"40127"
],
"CloudFront-Viewer-Country": [
"US"
],
"Host": [
"effmuaxft2.execute-api.us-east-1.amazonaws.com"
],
"Postman-Token": [
"10f731c3-bb8d-4720-971c-7fd5e7cf0065"
],
"User-Agent": [
"PostmanRuntime/7.31.3"
],
"Via": [
"1.1 3ae9464b3a12f9a00e97e3c81ee98466.cloudfront.net (CloudFront)"
],
"X-Amz-Cf-Id": [
"JWyqc8QN24nqpUdGuej_gc35vpExxV017XJD8EXxM8x1OupUzCGYbg=="
],
"X-Amzn-Trace-Id": [
"Root=1-641c7f2c-5c612f3e389094297688efa0"
],
"X-Forwarded-For": [
"134.174.21.156, 70.132.21.75"
],
"X-Forwarded-Port": [
"443"
],
"X-Forwarded-Proto": [
"https"
]
},
"queryStringParameters": {
"column": "gender"
},
"multiValueQueryStringParameters": {
"column": [
"gender"
]
},
"pathParameters": {
"subscription_name": "covid__encounter"
},
"stageVariables": "None",
"requestContext": {
"resourceId": "ta1ris",
"resourcePath": "/chart_data/{subscription_name}",
"httpMethod": "GET",
"extendedRequestId": "CPjPEHgQoAMFZ8A=",
"requestTime": "23/Mar/2023:16:32:44 +0000",
"path": "/dev/chart_data/covid__encounter",
"accountId": "316333106222",
"protocol": "HTTP/1.1",
"stage": "dev",
"domainPrefix": "effmuaxft2",
"requestTimeEpoch": 1679589164970,
"requestId": "ec3e7961-c978-45b5-af6b-46b70bcd9d7c",
"identity": {
"cognitoIdentityPoolId": "None",
"accountId": "None",
"cognitoIdentityId": "None",
"caller": "None",
"sourceIp": "134.174.21.156",
"principalOrgId": "None",
"accessKey": "None",
"cognitoAuthenticationType": "None",
"cognitoAuthenticationProvider": "None",
"userArn": "None",
"userAgent": "PostmanRuntime/7.31.3",
"user": "None"
},
"domainName": "effmuaxft2.execute-api.us-east-1.amazonaws.com",
"apiId": "effmuaxft2"
},
"body": "None",
"isBase64Encoded": false
}
46 changes: 46 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
[project]
name = "aggregator"
version = "0.1.0"
# This project is designed to run on the AWS serverless application framework (SAM).
# The project dependencies are handled via AWS layers. These are only required for
# local development.
dependencies= [
"awswrangler",
"boto3",
"pandas"
]
authors = [
{ name="Matt Garber", email="[email protected]" },
]
description = "Aggregates data from distributed hopsitals for display in Cumulus dashboard"
readme = "README.md"
license = { text="Apache License 2.0" }
dogversioning marked this conversation as resolved.
Show resolved Hide resolved
classifiers = [
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3",
"Topic :: Software Development :: Libraries :: Python Modules",
]

[package]
include = ["src/*.py"]
dogversioning marked this conversation as resolved.
Show resolved Hide resolved

[project.urls]
"Homepage" = "https://github.com/smart-on-fhir/cumulus-aggregator"

[build-system]
build-backend = "setuptools.build_meta"
mikix marked this conversation as resolved.
Show resolved Hide resolved
requires = [
"setuptools ~=63.2.0",
"wheel ~=0.37.1",
]

[project.optional-dependencies]
test = [
"moto[s3,athena] == 4.1.5",
dogversioning marked this conversation as resolved.
Show resolved Hide resolved
]
dev = [
"pre-commit",
"black",
"pytest",
"pytest-mock"
]
11 changes: 7 additions & 4 deletions scripts/cumulus_upload_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,19 @@ def upload_file(args):
try:
api_client = boto3.client("apigateway")
res = api_client.get_rest_apis()
site_api_dict = list(
api_dict = list(
filter(
lambda x: "cumulus-aggregator-dev"
in x["tags"]["aws:cloudformation:stack-name"],
res["items"],
)
)
api_id = site_api_dict[0]["id"]
url = f"https://{api_id}.execute-api.us-east-1.amazonaws.com/dev/"

for api in api_dict:
print(api["name"])
dogversioning marked this conversation as resolved.
Show resolved Hide resolved
if api["name"] == "CumulusAggregatorSiteAPI":
url = (
f"https://{api['id']}.execute-api.us-east-1.amazonaws.com/dev/"
)
except:
print("No response recieved from AWS API gateway.")
exit(1)
Expand Down
15 changes: 8 additions & 7 deletions src/handlers/dashboard/get_chart_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ def _get_table_cols(table_name: str) -> List:
"""
s3_bucket_name = os.environ.get("BUCKET_NAME")
s3_key = (
f"{BucketPath.CSVAGGREGATE.value}/{table_name.split('_')[0]}"
f"/{table_name}/{table_name}_aggregate.csv"
f"{BucketPath.CSVAGGREGATE.value}/{table_name.split('__')[0]}"
f"/{table_name}/{table_name}__aggregate.csv"
)
s3_client = boto3.client("s3")
s3_iter = s3_client.get_object(
Expand Down Expand Up @@ -54,13 +54,13 @@ def _build_query(query_params: Dict, filters: List, path_params: Dict) -> str:
group_str = f"{query_params['stratifier']}, {group_str}"
columns.remove(query_params["stratifier"])
if len(columns) > 0:
coalesce_str = f"WHERE COALESCE ({','.join(columns)}) = '' AND"
coalesce_str = f"WHERE COALESCE ({','.join(columns)}) IS NOT Null AND"
dogversioning marked this conversation as resolved.
Show resolved Hide resolved
else:
coalesce_str = "WHERE"
query_str = (
f"SELECT {select_str} FROM {table} " # nosec
f"SELECT {select_str} FROM \"{os.environ.get('GLUE_DB_NAME')}\".\"{table}\" " # nosec
f"{coalesce_str} "
f"{query_params['column']} != '' {filter_str} "
f"{query_params['column']} IS NOT Null {filter_str} "
f"GROUP BY {group_str}"
)
return query_str
Expand Down Expand Up @@ -100,8 +100,9 @@ def chart_data_handler(event, context):
query = _build_query(query_params, filters, path_params)
df = awswrangler.athena.read_sql_query(
query,
database="cumulus-aggregator-db",
s3_output="s3://cumulus-aggregator-site-counts/awswrangler",
database=os.environ.get("GLUE_DB_NAME"),
s3_output=f"s3://{os.environ.get('BUCKET_NAME')}/awswrangler",
workgroup=os.environ.get("WORKGROUP_NAME"),
)
res = _format_payload(df, query_params, filters)
res = http_response(200, res)
Expand Down
Loading