From c422aa694d76deb3cb2baea4be6c3e887a1be607 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:12:06 -0800 Subject: [PATCH 01/51] Rename route package to api --- app/api/{route => api}/__init__.py | 0 app/api/{route => api}/healthcheck.py | 4 ++-- app/api/{route => api}/response.py | 2 +- app/api/{route => api}/route_utils.py | 0 app/api/{route => api}/schemas/__init__.py | 0 app/api/{route => api}/schemas/request_schema.py | 0 app/api/{route => api}/schemas/response_schema.py | 2 +- app/api/{route => api}/schemas/user_schemas.py | 2 +- app/api/{route/user_route.py => api/users.py} | 8 ++------ app/api/app.py | 6 +++--- .../monitoring-and-observability/logging-configuration.md | 2 +- 11 files changed, 11 insertions(+), 15 deletions(-) rename app/api/{route => api}/__init__.py (100%) rename app/api/{route => api}/healthcheck.py (92%) rename app/api/{route => api}/response.py (97%) rename app/api/{route => api}/route_utils.py (100%) rename app/api/{route => api}/schemas/__init__.py (100%) rename app/api/{route => api}/schemas/request_schema.py (100%) rename app/api/{route => api}/schemas/response_schema.py (95%) rename app/api/{route => api}/schemas/user_schemas.py (97%) rename app/api/{route/user_route.py => api/users.py} (90%) diff --git a/app/api/route/__init__.py b/app/api/api/__init__.py similarity index 100% rename from app/api/route/__init__.py rename to app/api/api/__init__.py diff --git a/app/api/route/healthcheck.py b/app/api/api/healthcheck.py similarity index 92% rename from app/api/route/healthcheck.py rename to app/api/api/healthcheck.py index 80b76f6a..4df442b2 100644 --- a/app/api/route/healthcheck.py +++ b/app/api/api/healthcheck.py @@ -7,8 +7,8 @@ from werkzeug.exceptions import ServiceUnavailable import api.adapters.db.flask_db as flask_db -from api.route import response -from api.route.schemas import request_schema +from api.api import response +from api.api.schemas import request_schema logger = logging.getLogger(__name__) diff --git a/app/api/route/response.py b/app/api/api/response.py similarity index 97% rename from app/api/route/response.py rename to app/api/api/response.py index 4c843a62..52b4e5d2 100644 --- a/app/api/route/response.py +++ b/app/api/api/response.py @@ -2,7 +2,7 @@ from typing import Optional from api.db.models.base import Base -from api.route.schemas import response_schema +from api.api.schemas import response_schema @dataclasses.dataclass diff --git a/app/api/route/route_utils.py b/app/api/api/route_utils.py similarity index 100% rename from app/api/route/route_utils.py rename to app/api/api/route_utils.py diff --git a/app/api/route/schemas/__init__.py b/app/api/api/schemas/__init__.py similarity index 100% rename from app/api/route/schemas/__init__.py rename to app/api/api/schemas/__init__.py diff --git a/app/api/route/schemas/request_schema.py b/app/api/api/schemas/request_schema.py similarity index 100% rename from app/api/route/schemas/request_schema.py rename to app/api/api/schemas/request_schema.py diff --git a/app/api/route/schemas/response_schema.py b/app/api/api/schemas/response_schema.py similarity index 95% rename from app/api/route/schemas/response_schema.py rename to app/api/api/schemas/response_schema.py index 6dd25c0a..9ae6f6ac 100644 --- a/app/api/route/schemas/response_schema.py +++ b/app/api/api/schemas/response_schema.py @@ -1,6 +1,6 @@ from apiflask import fields -from api.route.schemas import request_schema +from api.api.schemas import request_schema class ValidationErrorSchema(request_schema.OrderedSchema): diff --git a/app/api/route/schemas/user_schemas.py b/app/api/api/schemas/user_schemas.py similarity index 97% rename from app/api/route/schemas/user_schemas.py rename to app/api/api/schemas/user_schemas.py index c02267bd..f0d55cc7 100644 --- a/app/api/route/schemas/user_schemas.py +++ b/app/api/api/schemas/user_schemas.py @@ -2,7 +2,7 @@ from marshmallow import fields as marshmallow_fields from api.db.models import user_models -from api.route.schemas import request_schema +from api.api.schemas import request_schema ############## # Role Models diff --git a/app/api/route/user_route.py b/app/api/api/users.py similarity index 90% rename from app/api/route/user_route.py rename to app/api/api/users.py index 30cbd8a8..901fb3d8 100644 --- a/app/api/route/user_route.py +++ b/app/api/api/users.py @@ -8,8 +8,8 @@ import api.services.users as user_service from api.auth.api_key_auth import api_key_auth from api.db.models.user_models import User -from api.route import response -from api.route.schemas import user_schemas +from api.api import response +from api.api.schemas import user_schemas from api.services import users logger = logging.getLogger(__name__) @@ -18,14 +18,12 @@ user_blueprint = APIBlueprint("user", __name__, tag="User") -@user_blueprint.post("/v1/user") @user_blueprint.input(user_schemas.UserSchema) @user_blueprint.output(user_schemas.UserSchema, status_code=201) @user_blueprint.auth_required(api_key_auth) @flask_db.with_db_session def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> dict: """ - POST /v1/user """ user = user_service.create_user(db_session, user_params) @@ -38,7 +36,6 @@ def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> di return response.ApiResponse(message="Success", data=user).asdict() -@user_blueprint.patch("/v1/user/") # Allow partial updates. partial=true means requests that are missing # required fields will not be rejected. # https://marshmallow.readthedocs.io/en/stable/quickstart.html#partial-loading @@ -59,7 +56,6 @@ def user_patch( return response.ApiResponse(message="Success", data=user).asdict() -@user_blueprint.get("/v1/user/") @user_blueprint.output(user_schemas.UserSchema) @user_blueprint.auth_required(api_key_auth) @flask_db.with_db_session diff --git a/app/api/app.py b/app/api/app.py index 81c6c146..4e39a59d 100644 --- a/app/api/app.py +++ b/app/api/app.py @@ -11,9 +11,9 @@ import api.logging import api.logging.flask_logger as flask_logger from api.auth.api_key_auth import User, get_app_security_scheme -from api.route.healthcheck import healthcheck_blueprint -from api.route.schemas import response_schema -from api.route.user_route import user_blueprint +from api.api.healthcheck import healthcheck_blueprint +from api.api.schemas import response_schema +from api.api.users import user_blueprint logger = logging.getLogger(__name__) diff --git a/docs/app/monitoring-and-observability/logging-configuration.md b/docs/app/monitoring-and-observability/logging-configuration.md index c1eaa778..1a62e37b 100644 --- a/docs/app/monitoring-and-observability/logging-configuration.md +++ b/docs/app/monitoring-and-observability/logging-configuration.md @@ -12,7 +12,7 @@ We have two separate ways of formatting the logs which are controlled by the `LO ```json { - "name": "api.route.healthcheck", + "name": "api.api.healthcheck", "levelname": "INFO", "funcName": "healthcheck_get", "created": "1663261542.0465896", From 45b108a346746a5c578e7adaaaae253ed47e5909 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:12:21 -0800 Subject: [PATCH 02/51] Make REST resource name plural --- app/api/api/users.py | 4 ++++ app/tests/api/route/test_user_route.py | 26 +++++++++++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/app/api/api/users.py b/app/api/api/users.py index 901fb3d8..75dd9b36 100644 --- a/app/api/api/users.py +++ b/app/api/api/users.py @@ -18,12 +18,14 @@ user_blueprint = APIBlueprint("user", __name__, tag="User") +@user_blueprint.post("/v1/users") @user_blueprint.input(user_schemas.UserSchema) @user_blueprint.output(user_schemas.UserSchema, status_code=201) @user_blueprint.auth_required(api_key_auth) @flask_db.with_db_session def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> dict: """ + POST /v1/users """ user = user_service.create_user(db_session, user_params) @@ -36,6 +38,7 @@ def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> di return response.ApiResponse(message="Success", data=user).asdict() +@user_blueprint.patch("/v1/users/") # Allow partial updates. partial=true means requests that are missing # required fields will not be rejected. # https://marshmallow.readthedocs.io/en/stable/quickstart.html#partial-loading @@ -56,6 +59,7 @@ def user_patch( return response.ApiResponse(message="Success", data=user).asdict() +@user_blueprint.get("/v1/users/") @user_blueprint.output(user_schemas.UserSchema) @user_blueprint.auth_required(api_key_auth) @flask_db.with_db_session diff --git a/app/tests/api/route/test_user_route.py b/app/tests/api/route/test_user_route.py index fcdb2a96..00a8b80a 100644 --- a/app/tests/api/route/test_user_route.py +++ b/app/tests/api/route/test_user_route.py @@ -27,7 +27,7 @@ def base_request(): @pytest.fixture def created_user(client, api_auth_token, base_request): - response = client.post("/v1/user", json=base_request, headers={"X-Auth": api_auth_token}) + response = client.post("/v1/users", json=base_request, headers={"X-Auth": api_auth_token}) return response.get_json()["data"] @@ -44,7 +44,7 @@ def test_create_and_get_user(client, base_request, api_auth_token, roles): **base_request, "roles": roles, } - post_response = client.post("/v1/user", json=request, headers={"X-Auth": api_auth_token}) + post_response = client.post("/v1/users", json=request, headers={"X-Auth": api_auth_token}) post_response_data = post_response.get_json()["data"] expected_response = { **request, @@ -60,7 +60,7 @@ def test_create_and_get_user(client, base_request, api_auth_token, roles): # Get the user user_id = post_response.get_json()["data"]["id"] - get_response = client.get(f"/v1/user/{user_id}", headers={"X-Auth": api_auth_token}) + get_response = client.get(f"/v1/users/{user_id}", headers={"X-Auth": api_auth_token}) assert get_response.status_code == 200 @@ -117,7 +117,7 @@ def test_create_and_get_user(client, base_request, api_auth_token, roles): @pytest.mark.parametrize("request_data,expected_response_data", test_create_user_bad_request_data) def test_create_user_bad_request(client, api_auth_token, request_data, expected_response_data): - response = client.post("/v1/user", json=request_data, headers={"X-Auth": api_auth_token}) + response = client.post("/v1/users", json=request_data, headers={"X-Auth": api_auth_token}) assert response.status_code == 422 response_data = response.get_json()["detail"]["json"] @@ -128,7 +128,7 @@ def test_patch_user(client, api_auth_token, created_user): user_id = created_user["id"] patch_request = {"first_name": fake.first_name()} patch_response = client.patch( - f"/v1/user/{user_id}", json=patch_request, headers={"X-Auth": api_auth_token} + f"/v1/users/{user_id}", json=patch_request, headers={"X-Auth": api_auth_token} ) patch_response_data = patch_response.get_json()["data"] expected_response_data = { @@ -140,7 +140,7 @@ def test_patch_user(client, api_auth_token, created_user): assert patch_response.status_code == 200 assert patch_response_data == expected_response_data - get_response = client.get(f"/v1/user/{user_id}", headers={"X-Auth": api_auth_token}) + get_response = client.get(f"/v1/users/{user_id}", headers={"X-Auth": api_auth_token}) get_response_data = get_response.get_json()["data"] assert get_response_data == expected_response_data @@ -154,13 +154,13 @@ def test_patch_user_roles(client, base_request, api_auth_token, initial_roles, u "roles": initial_roles, } created_user = client.post( - "/v1/user", json=post_request, headers={"X-Auth": api_auth_token} + "/v1/users", json=post_request, headers={"X-Auth": api_auth_token} ).get_json()["data"] user_id = created_user["id"] patch_request = {"roles": updated_roles} patch_response = client.patch( - f"/v1/user/{user_id}", json=patch_request, headers={"X-Auth": api_auth_token} + f"/v1/users/{user_id}", json=patch_request, headers={"X-Auth": api_auth_token} ) patch_response_data = patch_response.get_json()["data"] expected_response_data = { @@ -172,16 +172,16 @@ def test_patch_user_roles(client, base_request, api_auth_token, initial_roles, u assert patch_response.status_code == 200 assert patch_response_data == expected_response_data - get_response = client.get(f"/v1/user/{user_id}", headers={"X-Auth": api_auth_token}) + get_response = client.get(f"/v1/users/{user_id}", headers={"X-Auth": api_auth_token}) get_response_data = get_response.get_json()["data"] assert get_response_data == expected_response_data test_unauthorized_data = [ - pytest.param("post", "/v1/user", get_base_request(), id="post"), - pytest.param("get", f"/v1/user/{uuid.uuid4()}", None, id="get"), - pytest.param("patch", f"/v1/user/{uuid.uuid4()}", {}, id="patch"), + pytest.param("post", "/v1/users", get_base_request(), id="post"), + pytest.param("get", f"/v1/users/{uuid.uuid4()}", None, id="get"), + pytest.param("patch", f"/v1/users/{uuid.uuid4()}", {}, id="patch"), ] @@ -205,7 +205,7 @@ def test_unauthorized(client, method, url, body, api_auth_token): @pytest.mark.parametrize("method,body", test_not_found_data) def test_not_found(client, api_auth_token, method, body): user_id = uuid.uuid4() - url = f"/v1/user/{user_id}" + url = f"/v1/users/{user_id}" response = getattr(client, method)(url, json=body, headers={"X-Auth": api_auth_token}) assert response.status_code == 404 From 54d0b332dcf6cb1f1068ed0f4f96587f9aa7fd1c Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:13:56 -0800 Subject: [PATCH 03/51] Remove unused route_utils module --- app/api/api/route_utils.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 app/api/api/route_utils.py diff --git a/app/api/api/route_utils.py b/app/api/api/route_utils.py deleted file mode 100644 index 19b1f1f0..00000000 --- a/app/api/api/route_utils.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import Type, TypeVar -from uuid import UUID - -from apiflask import HTTPError -from sqlalchemy.orm import scoped_session - -_T = TypeVar("_T") - - -def get_or_404(db_session: scoped_session, model: Type[_T], id: UUID | str | int) -> _T: - """ - Utility method for fetching a single record from the DB by - its primary key ID, and raising a NotFound exception if - no such record exists. - """ - result = db_session.query(model).get(id) - - if result is None: - raise HTTPError(404, message=f"Could not find {model.__name__} with ID {id}") - - return result From f2b7cd5dfef249c55ac72fd88f392d5fec685afc Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:22:35 -0800 Subject: [PATCH 04/51] Use consistent import styles --- app/api/api/users.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/api/api/users.py b/app/api/api/users.py index 75dd9b36..24c01475 100644 --- a/app/api/api/users.py +++ b/app/api/api/users.py @@ -5,12 +5,12 @@ import api.adapters.db as db import api.adapters.db.flask_db as flask_db +import api.api.response as response +import api.api.schemas.user_schemas as user_schemas import api.services.users as user_service +import api.services.users as users from api.auth.api_key_auth import api_key_auth from api.db.models.user_models import User -from api.api import response -from api.api.schemas import user_schemas -from api.services import users logger = logging.getLogger(__name__) From 56d2afcd25bbda08ff72aed66f64ec3941e36087 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:27:34 -0800 Subject: [PATCH 05/51] Remove extraneous print statement --- app/api/api/users.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/app/api/api/users.py b/app/api/api/users.py index 24c01475..a706588b 100644 --- a/app/api/api/users.py +++ b/app/api/api/users.py @@ -29,11 +29,7 @@ def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> di """ user = user_service.create_user(db_session, user_params) - logger.info( - "Successfully inserted user", - extra=get_user_log_params(user), - ) - print("Successfully inserted user", get_user_log_params(user)) + logger.info("Successfully inserted user", extra=get_user_log_params(user)) return response.ApiResponse(message="Success", data=user).asdict() @@ -51,10 +47,7 @@ def user_patch( ) -> dict: user = user_service.patch_user(db_session, user_id, patch_user_params) - logger.info( - "Successfully patched user", - extra=get_user_log_params(user), - ) + logger.info("Successfully patched user", extra=get_user_log_params(user)) return response.ApiResponse(message="Success", data=user).asdict() @@ -66,13 +59,10 @@ def user_patch( def user_get(db_session: db.Session, user_id: str) -> dict: user = user_service.get_user(db_session, user_id) - logger.info( - "Successfully fetched user", - extra=get_user_log_params(user), - ) + logger.info("Successfully fetched user", extra=get_user_log_params(user)) return response.ApiResponse(message="Success", data=user).asdict() def get_user_log_params(user: User) -> dict[str, Any]: - return {"user_id": user.id} + return {"user.id": user.id} From 6fefac1b6f82813744b13d7052bab2d5d853cf72 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:28:16 -0800 Subject: [PATCH 06/51] Remove extra whitespace --- app/api/api/users.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/api/api/users.py b/app/api/api/users.py index a706588b..c322ad99 100644 --- a/app/api/api/users.py +++ b/app/api/api/users.py @@ -28,9 +28,7 @@ def user_post(db_session: db.Session, user_params: users.CreateUserParams) -> di POST /v1/users """ user = user_service.create_user(db_session, user_params) - logger.info("Successfully inserted user", extra=get_user_log_params(user)) - return response.ApiResponse(message="Success", data=user).asdict() @@ -46,9 +44,7 @@ def user_patch( db_session: db.Session, user_id: str, patch_user_params: users.PatchUserParams ) -> dict: user = user_service.patch_user(db_session, user_id, patch_user_params) - logger.info("Successfully patched user", extra=get_user_log_params(user)) - return response.ApiResponse(message="Success", data=user).asdict() @@ -58,9 +54,7 @@ def user_patch( @flask_db.with_db_session def user_get(db_session: db.Session, user_id: str) -> dict: user = user_service.get_user(db_session, user_id) - logger.info("Successfully fetched user", extra=get_user_log_params(user)) - return response.ApiResponse(message="Success", data=user).asdict() From c9630b672b7a2b910490a7538f116116ba000bcb Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 16:41:37 -0800 Subject: [PATCH 07/51] Reorganize user blueprint --- app/api/api/users/__init__.py | 4 ++++ app/api/api/users/user_blueprint.py | 4 ++++ app/api/api/{users.py => users/user_routes.py} | 8 ++------ app/api/api/{schemas => users}/user_schemas.py | 9 --------- 4 files changed, 10 insertions(+), 15 deletions(-) create mode 100644 app/api/api/users/__init__.py create mode 100644 app/api/api/users/user_blueprint.py rename app/api/api/{users.py => users/user_routes.py} (93%) rename app/api/api/{schemas => users}/user_schemas.py (94%) diff --git a/app/api/api/users/__init__.py b/app/api/api/users/__init__.py new file mode 100644 index 00000000..451cb95d --- /dev/null +++ b/app/api/api/users/__init__.py @@ -0,0 +1,4 @@ +import api.api.users.user_routes +from api.api.users.user_blueprint import user_blueprint + +__all__ = ["user_blueprint"] diff --git a/app/api/api/users/user_blueprint.py b/app/api/api/users/user_blueprint.py new file mode 100644 index 00000000..373b08a7 --- /dev/null +++ b/app/api/api/users/user_blueprint.py @@ -0,0 +1,4 @@ +from apiflask import APIBlueprint + + +user_blueprint = APIBlueprint("user", __name__, tag="User") diff --git a/app/api/api/users.py b/app/api/api/users/user_routes.py similarity index 93% rename from app/api/api/users.py rename to app/api/api/users/user_routes.py index c322ad99..5ef77eee 100644 --- a/app/api/api/users.py +++ b/app/api/api/users/user_routes.py @@ -1,23 +1,19 @@ import logging from typing import Any -from apiflask import APIBlueprint - import api.adapters.db as db import api.adapters.db.flask_db as flask_db import api.api.response as response -import api.api.schemas.user_schemas as user_schemas +import api.api.users.user_schemas as user_schemas import api.services.users as user_service import api.services.users as users +from api.api.users.user_blueprint import user_blueprint from api.auth.api_key_auth import api_key_auth from api.db.models.user_models import User logger = logging.getLogger(__name__) -user_blueprint = APIBlueprint("user", __name__, tag="User") - - @user_blueprint.post("/v1/users") @user_blueprint.input(user_schemas.UserSchema) @user_blueprint.output(user_schemas.UserSchema, status_code=201) diff --git a/app/api/api/schemas/user_schemas.py b/app/api/api/users/user_schemas.py similarity index 94% rename from app/api/api/schemas/user_schemas.py rename to app/api/api/users/user_schemas.py index f0d55cc7..de59be85 100644 --- a/app/api/api/schemas/user_schemas.py +++ b/app/api/api/users/user_schemas.py @@ -4,10 +4,6 @@ from api.db.models import user_models from api.api.schemas import request_schema -############## -# Role Models -############## - class RoleSchema(request_schema.OrderedSchema): type = marshmallow_fields.Enum( @@ -20,11 +16,6 @@ class RoleSchema(request_schema.OrderedSchema): # will always be a nested fields of the API user -############## -# User Models -############## - - class UserSchema(request_schema.OrderedSchema): id = fields.UUID(dump_only=True) first_name = fields.String(metadata={"description": "The user's first name"}, required=True) From 6d1d7df770a14ea4d62c14573fb7ef83b78ab6cd Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 17:20:37 -0800 Subject: [PATCH 08/51] Update apiflask --- app/poetry.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/poetry.lock b/app/poetry.lock index 6bf1119e..bd7f28d3 100644 --- a/app/poetry.lock +++ b/app/poetry.lock @@ -21,14 +21,14 @@ tz = ["python-dateutil"] [[package]] name = "apiflask" -version = "1.2.0" +version = "1.2.1" description = "A lightweight web API framework based on Flask and marshmallow-code projects." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "APIFlask-1.2.0-py3-none-any.whl", hash = "sha256:e93b3323d7cc62db672c59a9b425c768b52a1d626e5ef2e5f5f86b254012e19e"}, - {file = "APIFlask-1.2.0.tar.gz", hash = "sha256:0bb24d79cc8e381673d9ef7d3b5cfd299e6813f9c086d5d372d97116768e251e"}, + {file = "APIFlask-1.2.1-py3-none-any.whl", hash = "sha256:71452cb2dee41053f8cd1e7dd7d976ba9deb54f1a76845e375c1ca5af3c7c794"}, + {file = "APIFlask-1.2.1.tar.gz", hash = "sha256:505431a2c61ec3fc7e8fdb81f653d09c1ccec5b1e4e94f878f730af70a31be54"}, ] [package.dependencies] @@ -1579,7 +1579,7 @@ sqlalchemy2-stubs = {version = "*", optional = true, markers = "extra == \"mypy\ [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] @@ -1589,14 +1589,14 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "sqlalchemy2-stubs" From bb64fa317692a10a1fd69c1bd737de4a512b2543 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 17:25:47 -0800 Subject: [PATCH 09/51] Add create-csv command --- app/api/api/users/__init__.py | 1 + app/api/api/users/user_blueprint.py | 2 +- app/api/api/users/user_commands.py | 30 +++++++++++++++++++++++++++++ app/api/services/users/__init__.py | 2 ++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 app/api/api/users/user_commands.py diff --git a/app/api/api/users/__init__.py b/app/api/api/users/__init__.py index 451cb95d..0005ad7d 100644 --- a/app/api/api/users/__init__.py +++ b/app/api/api/users/__init__.py @@ -1,4 +1,5 @@ import api.api.users.user_routes +import api.api.users.user_commands from api.api.users.user_blueprint import user_blueprint __all__ = ["user_blueprint"] diff --git a/app/api/api/users/user_blueprint.py b/app/api/api/users/user_blueprint.py index 373b08a7..73a1d1c4 100644 --- a/app/api/api/users/user_blueprint.py +++ b/app/api/api/users/user_blueprint.py @@ -1,4 +1,4 @@ from apiflask import APIBlueprint -user_blueprint = APIBlueprint("user", __name__, tag="User") +user_blueprint = APIBlueprint("user", __name__, tag="User", cli_group="user") diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py new file mode 100644 index 00000000..24588766 --- /dev/null +++ b/app/api/api/users/user_commands.py @@ -0,0 +1,30 @@ +import logging +import os + +from flask.cli import AppGroup + +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db +import api.services.users as user_service +from api.api.users.user_blueprint import user_blueprint +from api.util.datetime_util import utcnow + +logger = logging.getLogger(__name__) +user_cli = AppGroup("user") + + +@user_blueprint.cli.command("create-csv") +@flask_db.with_db_session +def create_csv(db_session: db.Session): + # Build the path for the output file + # This will create a file in the folder specified like: + # s3://your-bucket/path/to/2022-09-09-12-00-00-user-roles.csv + # The file path can be either S3 or local disk. + output_path = os.getenv("USER_ROLE_CSV_OUTPUT_PATH", None) + if not output_path: + raise Exception("Please specify an USER_ROLE_CSV_OUTPUT_PATH env var") + + file_name = utcnow().strftime("%Y-%m-%d-%H-%M-%S") + "-user-roles.csv" + output_file_path = os.path.join(output_path, file_name) + + user_service.create_user_csv(db_session, output_file_path) diff --git a/app/api/services/users/__init__.py b/app/api/services/users/__init__.py index c550e936..16ad56f2 100644 --- a/app/api/services/users/__init__.py +++ b/app/api/services/users/__init__.py @@ -1,6 +1,7 @@ from .create_user import CreateUserParams, RoleParams, create_user from .get_user import get_user from .patch_user import PatchUserParams, patch_user +from .create_user_csv import create_user_csv __all__ = [ "CreateUserParams", @@ -9,4 +10,5 @@ "create_user", "get_user", "patch_user", + "create_user_csv", ] From a373ccb193250fe94a29f213bc8b442f8ae486bb Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 17:28:14 -0800 Subject: [PATCH 10/51] Update openapi.yml --- app/openapi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/openapi.yml b/app/openapi.yml index c5832aab..b935bc5f 100644 --- a/app/openapi.yml +++ b/app/openapi.yml @@ -46,7 +46,7 @@ paths: tags: - Health summary: Health - /v1/user: + /v1/users: post: parameters: [] responses: @@ -87,7 +87,7 @@ paths: description: Authentication error tags: - User - summary: POST /v1/user + summary: POST /v1/users requestBody: content: application/json: @@ -95,7 +95,7 @@ paths: $ref: '#/components/schemas/User' security: - ApiKeyAuth: [] - /v1/user/{user_id}: + /v1/users/{user_id}: get: parameters: - in: path From 941cb064ca6b3cfb28187a2674f80b68eb39471a Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 18:17:19 -0800 Subject: [PATCH 11/51] Move local.env to .env.template --- app/{local.env => .env.template} | 5 +++++ docker-compose.yml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) rename app/{local.env => .env.template} (92%) diff --git a/app/local.env b/app/.env.template similarity index 92% rename from app/local.env rename to app/.env.template index 71961e47..a04d69a5 100644 --- a/app/local.env +++ b/app/.env.template @@ -17,6 +17,11 @@ PYTHONPATH=/app/ FLASK_APP=api.app:create_app +# Setting FLASK_ENV to development will enable debug mode. +# flask run will use the interactive debugger and reloader by default in debug mode +FLASK_DEBUG=True +FLASK_RUN_PORT=8080 + ############################ # Logging ############################ diff --git a/docker-compose.yml b/docker-compose.yml index 13bcc57e..f2041982 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: command: postgres -c "log_lock_waits=on" -N 1000 -c "fsync=off" # Load environment variables for local development. - env_file: ./app/local.env + env_file: ./app/.env ports: - "5432:5432" volumes: @@ -28,7 +28,7 @@ services: container_name: main-app # Load environment variables for local development - env_file: ./app/local.env + env_file: ./app/.env # NOTE: These values take precedence if the same value is specified in the env_file. environment: # The env_file defines DB_HOST=localhost for accessing a non-dockerized database. From 42b5d49cbe2725c9655f409056a759b5ccdbf0a9 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 18:17:32 -0800 Subject: [PATCH 12/51] Remove unused poetry command --- app/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/app/pyproject.toml b/app/pyproject.toml index b67bbe61..90ac475d 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -46,7 +46,6 @@ build-backend = "poetry.core.masonry.api" db-migrate-up = "api.db.migrations.run:up" db-migrate-down = "api.db.migrations.run:down" db-migrate-down-all = "api.db.migrations.run:downall" -create-user-csv = "api.scripts.create_user_csv:main" [tool.black] line-length = 100 From cc9654b8dab5465fa8997b541af731a35010801e Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 18:17:39 -0800 Subject: [PATCH 13/51] Add Makefile command for Flask CLI --- app/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Makefile b/app/Makefile index 73709987..2474fb9e 100644 --- a/app/Makefile +++ b/app/Makefile @@ -171,11 +171,11 @@ lint-security: # https://bandit.readthedocs.io/en/latest/index.html ################################################## -# Scripts +# Flask CLI Custom Commands ################################################## -create-user-csv: - $(PY_RUN_CMD) create-user-csv +cli: + $(PY_RUN_CMD) flask $(args) ################################################## # Miscellaneous Utilities From 1e365e1a62730d431d94bf14da5ec1c3c2ae266d Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 18:24:02 -0800 Subject: [PATCH 14/51] Add help messages --- app/api/api/users/user_commands.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py index 24588766..b383ed5c 100644 --- a/app/api/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -1,8 +1,6 @@ import logging import os -from flask.cli import AppGroup - import api.adapters.db as db import api.adapters.db.flask_db as flask_db import api.services.users as user_service @@ -10,10 +8,11 @@ from api.util.datetime_util import utcnow logger = logging.getLogger(__name__) -user_cli = AppGroup("user") + +user_blueprint.cli.help = "User commands" -@user_blueprint.cli.command("create-csv") +@user_blueprint.cli.command("create-csv", help="Create a CSV of all users and their roles") @flask_db.with_db_session def create_csv(db_session: db.Session): # Build the path for the output file From 95be2d7bbd95a7c3d36baefdc9f169d020077acc Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 8 Feb 2023 18:27:01 -0800 Subject: [PATCH 15/51] Add make setup-local --- app/{.env.template => .env.example} | 0 app/Makefile | 7 +++++++ docs/app/getting-started.md | 11 ++++++----- 3 files changed, 13 insertions(+), 5 deletions(-) rename app/{.env.template => .env.example} (100%) diff --git a/app/.env.template b/app/.env.example similarity index 100% rename from app/.env.template rename to app/.env.example diff --git a/app/Makefile b/app/Makefile index 2474fb9e..18355bfa 100644 --- a/app/Makefile +++ b/app/Makefile @@ -33,6 +33,13 @@ else PY_RUN_CMD := docker-compose run $(DOCKER_EXEC_ARGS) --rm $(APP_NAME) poetry run endif +################################################## +# Local Development Setup +################################################## + +setup-local: + cp .env.example .env + ################################################## # API Build & Run ################################################## diff --git a/docs/app/getting-started.md b/docs/app/getting-started.md index 99f7857e..444efb10 100644 --- a/docs/app/getting-started.md +++ b/docs/app/getting-started.md @@ -31,12 +31,13 @@ make setup-local ## Run the application -1. In your terminal, `cd` to the app directory of this repo. +1. In your terminal, `cd` to the `app` directory of this repo. 2. Make sure you have [Docker Desktop](https://www.docker.com/products/docker-desktop/) installed & running. -3. Run `make init start` to build the image and start the container. -4. Navigate to `localhost:8080/v1/docs` to access the Swagger UI. -5. Run `make run-logs` to see the logs of the running API container -6. Run `make stop` when you are done to delete the container. +3. Run `make setup-local` to copy over `.env.example` to `.env` +4. Run `make init start` to build the image and start the container. +5. Navigate to `localhost:8080/v1/docs` to access the Swagger UI. +6. Run `make run-logs` to see the logs of the running API container +7. Run `make stop` when you are done to delete the container. ## Next steps From 6d65a046abe8210b4124b5bd08597216696dd3e0 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 9 Feb 2023 14:20:38 -0800 Subject: [PATCH 16/51] Revert "Move local.env to .env.template" This reverts commit 941cb064ca6b3cfb28187a2674f80b68eb39471a. --- app/{.env.example => local.env} | 5 ----- docker-compose.yml | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) rename app/{.env.example => local.env} (92%) diff --git a/app/.env.example b/app/local.env similarity index 92% rename from app/.env.example rename to app/local.env index a04d69a5..71961e47 100644 --- a/app/.env.example +++ b/app/local.env @@ -17,11 +17,6 @@ PYTHONPATH=/app/ FLASK_APP=api.app:create_app -# Setting FLASK_ENV to development will enable debug mode. -# flask run will use the interactive debugger and reloader by default in debug mode -FLASK_DEBUG=True -FLASK_RUN_PORT=8080 - ############################ # Logging ############################ diff --git a/docker-compose.yml b/docker-compose.yml index f2041982..13bcc57e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: command: postgres -c "log_lock_waits=on" -N 1000 -c "fsync=off" # Load environment variables for local development. - env_file: ./app/.env + env_file: ./app/local.env ports: - "5432:5432" volumes: @@ -28,7 +28,7 @@ services: container_name: main-app # Load environment variables for local development - env_file: ./app/.env + env_file: ./app/local.env # NOTE: These values take precedence if the same value is specified in the env_file. environment: # The env_file defines DB_HOST=localhost for accessing a non-dockerized database. From 75ca748c49d8d4b59992892a0999e4a1ae3fcef7 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 9 Feb 2023 14:25:42 -0800 Subject: [PATCH 17/51] Reorganize Makefile commands --- app/Makefile | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/app/Makefile b/app/Makefile index 18355bfa..0f475cb9 100644 --- a/app/Makefile +++ b/app/Makefile @@ -33,12 +33,18 @@ else PY_RUN_CMD := docker-compose run $(DOCKER_EXEC_ARGS) --rm $(APP_NAME) poetry run endif +FLASK_CMD := $(PY_RUN_CMD) flask --env-file local.env + ################################################## -# Local Development Setup +# Local Development Environment Setup ################################################## setup-local: - cp .env.example .env + # Configure poetry to use virtualenvs in the project directory + poetry config virtualenvs.in-project true + + # Install dependencies + poetry install --no-root ################################################## # API Build & Run @@ -64,13 +70,6 @@ stop: check: format-check lint test -# Set init-db as pre-requisite since there seems to be a race condition -# where the DB can't yet receive connections if it's starting from a -# clean state (e.g. after make stop, make clean-volumes, make openapi-spec) -openapi-spec: init-db ## Generate OpenAPI spec - $(PY_RUN_CMD) flask spec --format yaml --output ./openapi.yml - - ################################################## # DB & migrations ################################################## @@ -178,11 +177,18 @@ lint-security: # https://bandit.readthedocs.io/en/latest/index.html ################################################## -# Flask CLI Custom Commands +# Flask CLI Commands ################################################## cli: - $(PY_RUN_CMD) flask $(args) + $(FLASK_CMD) $(args) + +# Set init-db as pre-requisite since there seems to be a race condition +# where the DB can't yet receive connections if it's starting from a +# clean state (e.g. after make stop, make clean-volumes, make openapi-spec) +openapi-spec: init-db ## Generate OpenAPI spec + $(FLASK_CMD) spec --format yaml --output ./openapi.yml + ################################################## # Miscellaneous Utilities @@ -194,10 +200,3 @@ login: start ## Start shell in running container # Pauses for 5 seconds sleep-5: sleep 5 - -setup-local: - # Configure poetry to use virtualenvs in the project directory - poetry config virtualenvs.in-project true - - # Install dependencies - poetry install --no-root From 716128fa25271cbf973f9e0236a2c1fe14e3a80a Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 9 Feb 2023 14:30:29 -0800 Subject: [PATCH 18/51] Add comment --- app/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/Makefile b/app/Makefile index 0f475cb9..0b515535 100644 --- a/app/Makefile +++ b/app/Makefile @@ -177,12 +177,15 @@ lint-security: # https://bandit.readthedocs.io/en/latest/index.html ################################################## -# Flask CLI Commands +# CLI Commands ################################################## -cli: +cli: ## Run Flask app CLI command (Run make cli args="--help" to see list of CLI commands) $(FLASK_CMD) $(args) +user-create-csv: + $(FLASK_CMD) user create-csv + # Set init-db as pre-requisite since there seems to be a race condition # where the DB can't yet receive connections if it's starting from a # clean state (e.g. after make stop, make clean-volumes, make openapi-spec) From 78da5d2dd5a3f1f3fab0b7adb086f515529fb1f0 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 9 Feb 2023 14:35:14 -0800 Subject: [PATCH 19/51] Whitespace --- app/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Makefile b/app/Makefile index 0b515535..28675dda 100644 --- a/app/Makefile +++ b/app/Makefile @@ -59,7 +59,6 @@ start: run-logs: start docker-compose logs --follow --no-color $(APP_NAME) - init: build init-db clean-volumes: ## Remove project docker volumes (which includes the DB state) From dfe2dd32776bd8ef6c9d9415eea3791bcb4a01fd Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 9 Feb 2023 14:42:38 -0800 Subject: [PATCH 20/51] Add cli_runner fixture --- app/tests/conftest.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index af80dc63..39c0e1c7 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,5 +1,7 @@ import logging +import flask +import flask.testing import _pytest.monkeypatch import boto3 import moto @@ -140,15 +142,20 @@ def isolated_db_factories_session(monkeypatch, isolated_db: db.DBClient) -> db.S # Make app session scoped so the database connection pool is only created once # for the test session. This speeds up the tests. @pytest.fixture(scope="session") -def app(): +def app() -> flask.Flask: return app_entry.create_app() @pytest.fixture -def client(app): +def client(app: flask.Flask) -> flask.testing.FlaskClient: return app.test_client() +@pytest.fixture +def cli_runner(app: flask.Flask) -> flask.testing.CliRunner: + return app.test_cli_runner() + + @pytest.fixture def api_auth_token(monkeypatch): auth_token = "abcd1234" From 11096e83c87e4a126366b73c4ff7ea953db49c69 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 06:43:35 -0800 Subject: [PATCH 21/51] Move isolated db stuff to create user csv --- app/api/api/users/user_commands.py | 24 ++--- app/tests/api/scripts/test_create_user_csv.py | 97 +++++++++++++------ app/tests/conftest.py | 29 +----- 3 files changed, 76 insertions(+), 74 deletions(-) diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py index b383ed5c..61eda3b3 100644 --- a/app/api/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -1,5 +1,7 @@ +from typing import Optional +import click import logging -import os +import os.path as path import api.adapters.db as db import api.adapters.db.flask_db as flask_db @@ -14,16 +16,10 @@ @user_blueprint.cli.command("create-csv", help="Create a CSV of all users and their roles") @flask_db.with_db_session -def create_csv(db_session: db.Session): - # Build the path for the output file - # This will create a file in the folder specified like: - # s3://your-bucket/path/to/2022-09-09-12-00-00-user-roles.csv - # The file path can be either S3 or local disk. - output_path = os.getenv("USER_ROLE_CSV_OUTPUT_PATH", None) - if not output_path: - raise Exception("Please specify an USER_ROLE_CSV_OUTPUT_PATH env var") - - file_name = utcnow().strftime("%Y-%m-%d-%H-%M-%S") + "-user-roles.csv" - output_file_path = os.path.join(output_path, file_name) - - user_service.create_user_csv(db_session, output_file_path) +@click.option("--dir", default=".") +@click.option("--filename", default=None) +def create_csv(db_session: db.Session, dir: str, filename: str): + if filename is None: + filename = utcnow().strftime("%Y-%m-%d-%H-%M-%S") + "-user-roles.csv" + filepath = path.join(dir, filename) + user_service.create_user_csv(db_session, filepath) diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 0b2a24d7..b36c8dad 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -1,18 +1,61 @@ import csv import os +import flask.testing import pytest from smart_open import open as smart_open +import api.adapters.db as db +import api.app as app_entry +import tests.api.db.models.factories as factories +from api.db import models +from api.db.models.user_models import User from api.services.users.create_user_csv import USER_CSV_RECORD_HEADERS, create_user_csv from api.util.file_util import list_files from api.util.string_utils import blank_for_null from tests.api.db.models.factories import UserFactory +from tests.lib import db_testing @pytest.fixture -def tmp_file_path(tmp_path): - return tmp_path / "example_file.csv" +def isolated_db(monkeypatch) -> db.DBClient: + """ + Creates an isolated database for the test function. + + Creates a new empty PostgreSQL schema, creates all tables in the new schema + using SQLAlchemy, then returns a db.DB instance that can be used to + get connections or sessions to this database schema. The schema is dropped + after the test function completes. + + This is similar to the db fixture except the scope of the schema is the + individual test rather the test session. + """ + + with db_testing.create_isolated_db(monkeypatch) as db: + models.metadata.create_all(bind=db.get_connection()) + yield db + + +@pytest.fixture +def cli_runner(isolated_db) -> flask.testing.CliRunner: + """Overrides cli_runner from conftest to run in an isolated db schema""" + return app_entry.create_app().test_cli_runner() + + +@pytest.fixture +def isolated_db_factories_session(monkeypatch, isolated_db: db.DBClient) -> db.Session: + with isolated_db.get_session() as session: + monkeypatch.setattr(factories, "_db_session", session) + yield session + + +@pytest.fixture +def prepopulated_users(isolated_db_factories_session) -> list[User]: + return [ + UserFactory.create(first_name="A"), + UserFactory.create(first_name="B"), + UserFactory.create(first_name="C"), + ] def read_csv_records(file_path): @@ -42,38 +85,28 @@ def validate_csv_records(db_records, csv_records): ) -def test_create_user_csv_s3(isolated_db_factories_session, mock_s3_bucket): +def test_create_user_csv_s3( + cli_runner: flask.testing.FlaskCliRunner, prepopulated_users: list[User], mock_s3_bucket: str +): s3_filepath = f"s3://{mock_s3_bucket}/path/to/test.csv" - # To make validating these easier in the CSV, make the names consistent - db_records = [ - UserFactory.create(first_name="A"), - UserFactory.create(first_name="B"), - ] - - create_user_csv(isolated_db_factories_session, s3_filepath) - csv_rows = read_csv_records(s3_filepath) - validate_csv_records(db_records, csv_rows) - - # If we add another DB record it'll go in the file as well - db_records.append(UserFactory.create(first_name="C")) - create_user_csv(isolated_db_factories_session, s3_filepath) + cli_runner.invoke( + args=[ + "user", + "create-csv", + "--dir", + f"s3://{mock_s3_bucket}/path/to/", + "--filename", + "test.csv", + ] + ) csv_rows = read_csv_records(s3_filepath) - validate_csv_records(db_records, csv_rows) - - assert "test.csv" in list_files(f"s3://{mock_s3_bucket}/path/to/") - - -def test_create_user_csv_local(isolated_db_factories_session, tmp_path, tmp_file_path): - # Same as above test, but verifying the file logic - # works locally in addition to S3. - db_records = [ - UserFactory.create(first_name="A"), - UserFactory.create(first_name="B"), - ] + validate_csv_records(prepopulated_users, csv_rows) - create_user_csv(isolated_db_factories_session, tmp_file_path) - csv_rows = read_csv_records(tmp_file_path) - validate_csv_records(db_records, csv_rows) - assert os.path.exists(tmp_file_path) +def test_create_user_csv_local( + cli_runner: flask.testing.FlaskCliRunner, prepopulated_users: list[User], tmp_path +): + cli_runner.invoke(args=["user", "create-csv", "--dir", str(tmp_path), "--filename", "test.csv"]) + csv_rows = read_csv_records(os.path.join(tmp_path, "test.csv")) + validate_csv_records(prepopulated_users, csv_rows) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 39c0e1c7..4de145ed 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -65,25 +65,6 @@ def db_client(monkeypatch_session) -> db.DBClient: yield db_client -@pytest.fixture(scope="function") -def isolated_db(monkeypatch) -> db.DBClient: - """ - Creates an isolated database for the test function. - - Creates a new empty PostgreSQL schema, creates all tables in the new schema - using SQLAlchemy, then returns a db.DB instance that can be used to - get connections or sessions to this database schema. The schema is dropped - after the test function completes. - - This is similar to the db fixture except the scope of the schema is the - individual test rather the test session. - """ - - with db_testing.create_isolated_db(monkeypatch) as db: - models.metadata.create_all(bind=db.get_connection()) - yield db - - @pytest.fixture def empty_schema(monkeypatch) -> db.DBClient: """ @@ -126,14 +107,6 @@ def factories_db_session(monkeypatch, test_db_session) -> db.Session: return test_db_session -@pytest.fixture -def isolated_db_factories_session(monkeypatch, isolated_db: db.DBClient) -> db.Session: - with isolated_db.get_session() as session: - monkeypatch.setattr(factories, "_db_session", session) - logger.info("set factories db_session to %s", session) - yield session - - #################### # Test App & Client #################### @@ -142,7 +115,7 @@ def isolated_db_factories_session(monkeypatch, isolated_db: db.DBClient) -> db.S # Make app session scoped so the database connection pool is only created once # for the test session. This speeds up the tests. @pytest.fixture(scope="session") -def app() -> flask.Flask: +def app(db_client) -> flask.Flask: return app_entry.create_app() From 6e177cd6ed1b2455f63eaf2500ebcd782ff69d24 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 06:52:59 -0800 Subject: [PATCH 22/51] Install pytest lazy fixture --- app/poetry.lock | 19 +++++++++++++++++-- app/pyproject.toml | 1 + 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/app/poetry.lock b/app/poetry.lock index bd7f28d3..66907ebf 100644 --- a/app/poetry.lock +++ b/app/poetry.lock @@ -1308,6 +1308,21 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-lazy-fixture" +version = "0.6.3" +description = "It helps to use fixtures in pytest.mark.parametrize" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pytest-lazy-fixture-0.6.3.tar.gz", hash = "sha256:0e7d0c7f74ba33e6e80905e9bfd81f9d15ef9a790de97993e34213deb5ad10ac"}, + {file = "pytest_lazy_fixture-0.6.3-py3-none-any.whl", hash = "sha256:e0b379f38299ff27a653f03eaa69b08a6fd4484e46fd1c9907d984b9f9daeda6"}, +] + +[package.dependencies] +pytest = ">=3.2.5" + [[package]] name = "pytest-watch" version = "4.2.0" @@ -1820,6 +1835,6 @@ files = [ ] [metadata] -lock-version = "1.1" +lock-version = "2.0" python-versions = "^3.10" -content-hash = "1f654cfd2377d8065e73d9e96fe708fe13c76d5d789cd04702ace59d9e4b23a8" +content-hash = "01e8dc779c9db34a85e0c5ed33b87700a38feae540d9677eacf6ec5d6b8b4c7f" diff --git a/app/pyproject.toml b/app/pyproject.toml index 90ac475d..eb85e2cd 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -37,6 +37,7 @@ bandit = "^1.7.4" [tool.poetry.group.dev.dependencies] pytest-watch = "^4.2.0" +pytest-lazy-fixture = "^0.6.3" [build-system] requires = ["poetry-core>=1.0.0"] From 6aecce5034c14008dfa7a92a652fad660b5af266 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 07:37:14 -0800 Subject: [PATCH 23/51] Simplify assertion --- app/tests/api/scripts/test_create_user_csv.py | 87 +++++++------------ .../scripts/test_create_user_csv_expected.csv | 4 + 2 files changed, 34 insertions(+), 57 deletions(-) create mode 100644 app/tests/api/scripts/test_create_user_csv_expected.csv diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index b36c8dad..1d4ac08f 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -1,15 +1,15 @@ -import csv -import os +import os.path as path import flask.testing import pytest +from pytest_lazyfixture import lazy_fixture from smart_open import open as smart_open import api.adapters.db as db import api.app as app_entry import tests.api.db.models.factories as factories from api.db import models -from api.db.models.user_models import User +from api.db.models.user_models import User, Role from api.services.users.create_user_csv import USER_CSV_RECORD_HEADERS, create_user_csv from api.util.file_util import list_files from api.util.string_utils import blank_for_null @@ -52,61 +52,34 @@ def isolated_db_factories_session(monkeypatch, isolated_db: db.DBClient) -> db.S @pytest.fixture def prepopulated_users(isolated_db_factories_session) -> list[User]: return [ - UserFactory.create(first_name="A"), - UserFactory.create(first_name="B"), - UserFactory.create(first_name="C"), + UserFactory.create(first_name="Jon", last_name="Doe", is_active=True), + UserFactory.create(first_name="Jane", last_name="Doe", is_active=False), + UserFactory.create( + first_name="Alby", + last_name="Testin", + is_active=True, + ), ] -def read_csv_records(file_path): - with smart_open(file_path) as csv_file: - csv_reader = csv.DictReader(csv_file) - csv_rows = list(csv_reader) - return csv_rows - - -def validate_csv_records(db_records, csv_records): - - assert len(csv_records) == len(db_records) - - # Sort the two lists by name and zip together for validation - csv_records.sort(key=lambda record: record["User Name"]) - db_records.sort(key=lambda record: record.first_name) - for csv_record, db_record in zip(csv_records, db_records): - assert ( - csv_record[USER_CSV_RECORD_HEADERS.user_name] - == f"{db_record.first_name} {db_record.last_name}" - ) - assert csv_record[USER_CSV_RECORD_HEADERS.roles] == " ".join( - [role.type for role in db_record.roles] - ) - assert csv_record[USER_CSV_RECORD_HEADERS.is_user_active] == blank_for_null( - db_record.is_active - ) - - -def test_create_user_csv_s3( - cli_runner: flask.testing.FlaskCliRunner, prepopulated_users: list[User], mock_s3_bucket: str -): - s3_filepath = f"s3://{mock_s3_bucket}/path/to/test.csv" - - cli_runner.invoke( - args=[ - "user", - "create-csv", - "--dir", - f"s3://{mock_s3_bucket}/path/to/", - "--filename", - "test.csv", - ] - ) - csv_rows = read_csv_records(s3_filepath) - validate_csv_records(prepopulated_users, csv_rows) - - -def test_create_user_csv_local( - cli_runner: flask.testing.FlaskCliRunner, prepopulated_users: list[User], tmp_path +@pytest.fixture +def tmp_s3_folder(mock_s3_bucket): + return f"s3://{mock_s3_bucket}/path/to/" + + +@pytest.mark.parametrize( + "dir", + [ + pytest.param(lazy_fixture("tmp_s3_folder"), id="write-to-s3"), + pytest.param(lazy_fixture("tmp_path"), id="write-to-local"), + ], +) +def test_create_user_csv( + cli_runner: flask.testing.FlaskCliRunner, prepopulated_users: list[User], dir: str ): - cli_runner.invoke(args=["user", "create-csv", "--dir", str(tmp_path), "--filename", "test.csv"]) - csv_rows = read_csv_records(os.path.join(tmp_path, "test.csv")) - validate_csv_records(prepopulated_users, csv_rows) + cli_runner.invoke(args=["user", "create-csv", "--dir", dir, "--filename", "test.csv"]) + output = smart_open(path.join(dir, "test.csv")).read() + expected_output = open( + path.join(path.dirname(__file__), "test_create_user_csv_expected.csv") + ).read() + assert output == expected_output diff --git a/app/tests/api/scripts/test_create_user_csv_expected.csv b/app/tests/api/scripts/test_create_user_csv_expected.csv new file mode 100644 index 00000000..da0371ff --- /dev/null +++ b/app/tests/api/scripts/test_create_user_csv_expected.csv @@ -0,0 +1,4 @@ +"User Name","Roles","Is User Active?" +"Jon Doe","ADMIN USER","True" +"Jane Doe","ADMIN USER","False" +"Alby Testin","ADMIN USER","True" From 7053d911559f7a719d52bbe4dc550cd24b90ec38 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 07:39:18 -0800 Subject: [PATCH 24/51] Remove unused utils --- app/api/util/file_util.py | 61 --------- app/api/util/string_utils.py | 14 -- app/tests/api/scripts/test_create_user_csv.py | 4 +- app/tests/api/util/test_file_util.py | 124 ------------------ app/tests/api/util/test_string_utils.py | 21 --- 5 files changed, 1 insertion(+), 223 deletions(-) delete mode 100644 app/tests/api/util/test_file_util.py delete mode 100644 app/tests/api/util/test_string_utils.py diff --git a/app/api/util/file_util.py b/app/api/util/file_util.py index ced414f2..5972ba3a 100644 --- a/app/api/util/file_util.py +++ b/app/api/util/file_util.py @@ -45,64 +45,3 @@ def get_s3_client(boto_session: Optional[boto3.Session] = None) -> botocore.clie return boto_session.client("s3") return boto3.client("s3") - - -################################## -# File operation utils -################################## - - -def list_files( - path: str, recursive: bool = False, boto_session: Optional[boto3.Session] = None -) -> list[str]: - """List the immediate files under path. - - There is minor inconsistency between local path handling and S3 paths. - Directory names will be included for local paths, whereas they will not for - S3 paths. - - Args: - path: Supports s3:// and local paths. - recursive: Only applicable for S3 paths. - If set to True will recursively list all relative key paths under the prefix. - boto_session: Boto session object to use for S3 access. Only necessary - if needing to access an S3 bucket with assumed credentials (e.g., - cross-account bucket access). - """ - if is_s3_path(path): - bucket_name, prefix = split_s3_url(path) - - # in order for s3.list_objects to only list the immediate "files" under - # the given path, the prefix should end in the path delimiter - if prefix and not prefix.endswith("/"): - prefix = prefix + "/" - - s3 = get_s3_client(boto_session) - - # When the delimiter is provided, s3 knows to stop listing keys that contain it (starting after the prefix). - # https://docs.aws.amazon.com/AmazonS3/latest/dev/ListingKeysHierarchy.html - delimiter = "" if recursive else "/" - - paginator = s3.get_paginator("list_objects_v2") - pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix, Delimiter=delimiter) - - file_paths = [] - for page in pages: - object_contents = page.get("Contents") - - if object_contents: - for object in object_contents: - if recursive: - key = object["Key"] - start_index = key.index(prefix) + len(prefix) - file_paths.append(key[start_index:]) - else: - file_paths.append(get_file_name(object["Key"])) - - return file_paths - - # os.listdir throws an exception if the path doesn't exist - # Make it behave like S3 list and return an empty list - if os.path.exists(path): - return os.listdir(path) - return [] diff --git a/app/api/util/string_utils.py b/app/api/util/string_utils.py index c309f72b..367185a9 100644 --- a/app/api/util/string_utils.py +++ b/app/api/util/string_utils.py @@ -12,17 +12,3 @@ def join_list(joining_list: Optional[list], join_txt: str = "\n") -> str: return "" return join_txt.join(joining_list) - - -def blank_for_null(value: Optional[Any]) -> str: - """ - Utility to make a string blank if its null - - Functionally equivalent to - - ```"" if value is None else str(value)``` - """ - # If a value is blank - if value is None: - return "" - return str(value) diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 1d4ac08f..115ca783 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -9,10 +9,8 @@ import api.app as app_entry import tests.api.db.models.factories as factories from api.db import models -from api.db.models.user_models import User, Role +from api.db.models.user_models import User from api.services.users.create_user_csv import USER_CSV_RECORD_HEADERS, create_user_csv -from api.util.file_util import list_files -from api.util.string_utils import blank_for_null from tests.api.db.models.factories import UserFactory from tests.lib import db_testing diff --git a/app/tests/api/util/test_file_util.py b/app/tests/api/util/test_file_util.py deleted file mode 100644 index d8b5794a..00000000 --- a/app/tests/api/util/test_file_util.py +++ /dev/null @@ -1,124 +0,0 @@ -import os - -import pytest -from smart_open import open as smart_open - -import api.util.file_util as file_util - - -def create_file(root_path, file_path): - full_path = os.path.join(root_path, file_path) - - if not file_util.is_s3_path(str(full_path)): - os.makedirs(os.path.dirname(full_path), exist_ok=True) - - with smart_open(full_path, mode="w") as outfile: - outfile.write("hello") - - return full_path - - -@pytest.mark.parametrize( - "path,is_s3", - [ - ("s3://bucket/folder/test.txt", True), - ("./relative/folder/test.txt", False), - ("http://example.com/test.txt", False), - ], -) -def test_is_s3_path(path, is_s3): - assert file_util.is_s3_path(path) is is_s3 - - -@pytest.mark.parametrize( - "path,bucket,prefix", - [ - ("s3://my_bucket/my_key", "my_bucket", "my_key"), - ("s3://my_bucket/path/to/directory/", "my_bucket", "path/to/directory/"), - ("s3://my_bucket/path/to/file.txt", "my_bucket", "path/to/file.txt"), - ], -) -def test_split_s3_url(path, bucket, prefix): - assert file_util.split_s3_url(path) == (bucket, prefix) - - -@pytest.mark.parametrize( - "path,bucket", - [ - ("s3://bucket/folder/test.txt", "bucket"), - ("s3://bucket_x/folder", "bucket_x"), - ("s3://bucket-y/folder/", "bucket-y"), - ("s3://bucketz", "bucketz"), - ], -) -def test_get_s3_bucket(path, bucket): - assert file_util.get_s3_bucket(path) == bucket - - -@pytest.mark.parametrize( - "path,file_key", - [ - ("s3://bucket/folder/test.txt", "folder/test.txt"), - ("s3://bucket_x/file.csv", "file.csv"), - ("s3://bucket-y/folder/path/to/abc.zip", "folder/path/to/abc.zip"), - ("./folder/path", "/folder/path"), - ("sftp://folder/filename", "filename"), - ], -) -def test_get_s3_file_key(path, file_key): - assert file_util.get_s3_file_key(path) == file_key - - -@pytest.mark.parametrize( - "path,file_name", - [ - ("s3://bucket/folder/test.txt", "test.txt"), - ("s3://bucket_x/file.csv", "file.csv"), - ("s3://bucket-y/folder/path/to/abc.zip", "abc.zip"), - ("./folder/path", "path"), - ("sftp://filename", "filename"), - ], -) -def test_get_s3_file_name(path, file_name): - assert file_util.get_file_name(path) == file_name - - -def test_list_files_in_folder_fs(tmp_path): - create_file(tmp_path, "file1.txt") - create_file(tmp_path, "folder/file2.txt") - create_file(tmp_path, "different_folder/file3.txt") - create_file(tmp_path, "folder/nested_folder/file4.txt") - - assert "file1.txt" in file_util.list_files(tmp_path) - assert "file2.txt" in file_util.list_files(tmp_path / "folder") - assert "file3.txt" in file_util.list_files(tmp_path / "different_folder") - assert "file4.txt" in file_util.list_files(tmp_path / "folder/nested_folder") - - # Note that recursive doesn't work as implemented for the - # local filesystem, so no further testing is needed. - - -def test_list_files_in_folder_s3(mock_s3_bucket): - prefix = f"s3://{mock_s3_bucket}/" - create_file(prefix, "file1.txt") - create_file(prefix, "folder/file2.txt") - create_file(prefix, "different_folder/file3.txt") - create_file(prefix, "folder/nested_folder/file4.txt") - - assert "file1.txt" in file_util.list_files(prefix) - assert "file2.txt" in file_util.list_files(prefix + "folder") - assert "file3.txt" in file_util.list_files(prefix + "different_folder") - assert "file4.txt" in file_util.list_files(prefix + "folder/nested_folder") - - root_files_recursive = file_util.list_files(prefix, recursive=True) - assert set(root_files_recursive) == set( - [ - "file1.txt", - "folder/file2.txt", - "different_folder/file3.txt", - "folder/nested_folder/file4.txt", - ] - ) - - folder_files_recursive = file_util.list_files(prefix + "folder", recursive=True) - assert set(folder_files_recursive) == set(["file2.txt", "nested_folder/file4.txt"]) diff --git a/app/tests/api/util/test_string_utils.py b/app/tests/api/util/test_string_utils.py deleted file mode 100644 index 2390d715..00000000 --- a/app/tests/api/util/test_string_utils.py +++ /dev/null @@ -1,21 +0,0 @@ -from api.util.string_utils import blank_for_null, join_list - - -def test_join_list(): - assert join_list(None) == "" - assert join_list(None, ",") == "" - assert join_list(None, "|") == "" - assert join_list([]) == "" - assert join_list([], ",") == "" - assert join_list([], "|") == "" - - assert join_list(["a", "b", "c"]) == "a\nb\nc" - assert join_list(["a", "b", "c"], ",") == "a,b,c" - assert join_list(["a", "b", "c"], "|") == "a|b|c" - - -def test_blank_for_null(): - assert blank_for_null(None) == "" - assert blank_for_null("hello") == "hello" - assert blank_for_null(4) == "4" - assert blank_for_null(["a", "b"]) == "['a', 'b']" From 662ac67cb1943de130c0fc161bde40e9158d1a35 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 07:46:14 -0800 Subject: [PATCH 25/51] Lint --- app/api/api/response.py | 2 +- app/api/api/users/__init__.py | 9 +++++++-- app/api/api/users/user_blueprint.py | 1 - app/api/api/users/user_commands.py | 7 ++++--- app/api/api/users/user_schemas.py | 2 +- app/api/app.py | 2 +- app/api/services/users/__init__.py | 2 +- app/api/util/string_utils.py | 2 +- app/poetry.lock | 2 +- app/tests/api/scripts/test_create_user_csv.py | 1 - app/tests/conftest.py | 4 ++-- 11 files changed, 19 insertions(+), 15 deletions(-) diff --git a/app/api/api/response.py b/app/api/api/response.py index 52b4e5d2..65f6e0bb 100644 --- a/app/api/api/response.py +++ b/app/api/api/response.py @@ -1,8 +1,8 @@ import dataclasses from typing import Optional -from api.db.models.base import Base from api.api.schemas import response_schema +from api.db.models.base import Base @dataclasses.dataclass diff --git a/app/api/api/users/__init__.py b/app/api/api/users/__init__.py index 0005ad7d..1da0ac9f 100644 --- a/app/api/api/users/__init__.py +++ b/app/api/api/users/__init__.py @@ -1,5 +1,10 @@ -import api.api.users.user_routes -import api.api.users.user_commands from api.api.users.user_blueprint import user_blueprint +# import user_commands module to register the CLI commands on the user_blueprint +import api.api.users.user_commands # noqa: F401 E402 isort:skip + +# import user_commands module to register the API routes on the user_blueprint +import api.api.users.user_routes # noqa: F401 E402 isort:skip + + __all__ = ["user_blueprint"] diff --git a/app/api/api/users/user_blueprint.py b/app/api/api/users/user_blueprint.py index 73a1d1c4..3155fe4e 100644 --- a/app/api/api/users/user_blueprint.py +++ b/app/api/api/users/user_blueprint.py @@ -1,4 +1,3 @@ from apiflask import APIBlueprint - user_blueprint = APIBlueprint("user", __name__, tag="User", cli_group="user") diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py index 61eda3b3..20682822 100644 --- a/app/api/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -1,7 +1,8 @@ -from typing import Optional -import click import logging import os.path as path +from typing import Optional + +import click import api.adapters.db as db import api.adapters.db.flask_db as flask_db @@ -18,7 +19,7 @@ @flask_db.with_db_session @click.option("--dir", default=".") @click.option("--filename", default=None) -def create_csv(db_session: db.Session, dir: str, filename: str): +def create_csv(db_session: db.Session, dir: str, filename: Optional[str]) -> None: if filename is None: filename = utcnow().strftime("%Y-%m-%d-%H-%M-%S") + "-user-roles.csv" filepath = path.join(dir, filename) diff --git a/app/api/api/users/user_schemas.py b/app/api/api/users/user_schemas.py index de59be85..cc44c6b1 100644 --- a/app/api/api/users/user_schemas.py +++ b/app/api/api/users/user_schemas.py @@ -1,8 +1,8 @@ from apiflask import fields from marshmallow import fields as marshmallow_fields -from api.db.models import user_models from api.api.schemas import request_schema +from api.db.models import user_models class RoleSchema(request_schema.OrderedSchema): diff --git a/app/api/app.py b/app/api/app.py index 4e39a59d..4c118c90 100644 --- a/app/api/app.py +++ b/app/api/app.py @@ -10,10 +10,10 @@ import api.adapters.db.flask_db as flask_db import api.logging import api.logging.flask_logger as flask_logger -from api.auth.api_key_auth import User, get_app_security_scheme from api.api.healthcheck import healthcheck_blueprint from api.api.schemas import response_schema from api.api.users import user_blueprint +from api.auth.api_key_auth import User, get_app_security_scheme logger = logging.getLogger(__name__) diff --git a/app/api/services/users/__init__.py b/app/api/services/users/__init__.py index 16ad56f2..9d01b20f 100644 --- a/app/api/services/users/__init__.py +++ b/app/api/services/users/__init__.py @@ -1,7 +1,7 @@ from .create_user import CreateUserParams, RoleParams, create_user +from .create_user_csv import create_user_csv from .get_user import get_user from .patch_user import PatchUserParams, patch_user -from .create_user_csv import create_user_csv __all__ = [ "CreateUserParams", diff --git a/app/api/util/string_utils.py b/app/api/util/string_utils.py index 367185a9..9ad1b7e5 100644 --- a/app/api/util/string_utils.py +++ b/app/api/util/string_utils.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Optional def join_list(joining_list: Optional[list], join_txt: str = "\n") -> str: diff --git a/app/poetry.lock b/app/poetry.lock index 66907ebf..07193428 100644 --- a/app/poetry.lock +++ b/app/poetry.lock @@ -1835,6 +1835,6 @@ files = [ ] [metadata] -lock-version = "2.0" +lock-version = "1.1" python-versions = "^3.10" content-hash = "01e8dc779c9db34a85e0c5ed33b87700a38feae540d9677eacf6ec5d6b8b4c7f" diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 115ca783..fb315b35 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -10,7 +10,6 @@ import tests.api.db.models.factories as factories from api.db import models from api.db.models.user_models import User -from api.services.users.create_user_csv import USER_CSV_RECORD_HEADERS, create_user_csv from tests.api.db.models.factories import UserFactory from tests.lib import db_testing diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 4de145ed..8bd26567 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,9 +1,9 @@ import logging -import flask -import flask.testing import _pytest.monkeypatch import boto3 +import flask +import flask.testing import moto import pytest import sqlalchemy From 6daef3e9b44e8b8e8d0149e89ee6ef84aac5bd56 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 07:52:15 -0800 Subject: [PATCH 26/51] Add test for default filename --- app/tests/api/scripts/test_create_user_csv.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index fb315b35..ea615daa 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -1,4 +1,6 @@ +import os import os.path as path +import re import flask.testing import pytest @@ -80,3 +82,9 @@ def test_create_user_csv( path.join(path.dirname(__file__), "test_create_user_csv_expected.csv") ).read() assert output == expected_output + + +def test_default_filename(cli_runner: flask.testing.FlaskCliRunner, tmp_path: str): + cli_runner.invoke(args=["user", "create-csv", "--dir", tmp_path]) + filenames = os.listdir(tmp_path) + assert re.match(r"\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-user-roles.csv", filenames[0]) From 020c4cfa3d45bcbc68d11e34877edd9b70b199a2 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 07:52:41 -0800 Subject: [PATCH 27/51] Remove unused main function --- app/api/services/users/create_user_csv.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/app/api/services/users/create_user_csv.py b/app/api/services/users/create_user_csv.py index f365b2fd..eb4e5781 100644 --- a/app/api/services/users/create_user_csv.py +++ b/app/api/services/users/create_user_csv.py @@ -83,20 +83,3 @@ def convert_user_records_for_csv(records: list[User]) -> list[UserCsvRecord]: ) return out_records - - -def main() -> None: - # Initialize DB session / logging / env vars - with script_context_manager() as script_context: - # Build the path for the output file - # This will create a file in the folder specified like: - # s3://your-bucket/path/to/2022-09-09-12-00-00-user-roles.csv - # The file path can be either S3 or local disk. - output_path = os.getenv("USER_ROLE_CSV_OUTPUT_PATH", None) - if not output_path: - raise Exception("Please specify an USER_ROLE_CSV_OUTPUT_PATH env var") - - file_name = utcnow().strftime("%Y-%m-%d-%H-%M-%S") + "-user-roles.csv" - output_file_path = os.path.join(output_path, file_name) - - create_user_csv(script_context.db_session, output_file_path) From f1a29eba16d627d9f10c25b42157c0513a1095bf Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Sat, 11 Feb 2023 07:54:10 -0800 Subject: [PATCH 28/51] Remove script_util --- app/api/scripts/__init__.py | 0 app/api/scripts/util/__init__.py | 0 app/api/scripts/util/script_util.py | 41 ----------------------- app/api/services/users/create_user_csv.py | 4 --- 4 files changed, 45 deletions(-) delete mode 100644 app/api/scripts/__init__.py delete mode 100644 app/api/scripts/util/__init__.py delete mode 100644 app/api/scripts/util/script_util.py diff --git a/app/api/scripts/__init__.py b/app/api/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/api/scripts/util/__init__.py b/app/api/scripts/util/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/app/api/scripts/util/script_util.py b/app/api/scripts/util/script_util.py deleted file mode 100644 index 3cc5a2bb..00000000 --- a/app/api/scripts/util/script_util.py +++ /dev/null @@ -1,41 +0,0 @@ -# TODO use built in @flask.cli commands so that we can reuse flask app and no longer need this file -import logging -from contextlib import contextmanager -from dataclasses import dataclass -from typing import Generator - -import api.adapters.db as db -import api.logging -from api.util.local import load_local_env_vars - -logger = logging.getLogger(__name__) - - -@dataclass -class ScriptContext: - db_session: db.Session - - -# TODO remove -@contextmanager -def script_context_manager() -> Generator[ScriptContext, None, None]: - """ - Context manager for running a script - that needs to access the DB. Initializes - a few useful components like the DB connection, - logging, and local environment variables (if local). - """ - load_local_env_vars() - api.logging.init(__package__) - - # TODO - Note this is really basic, but - # it could a good place to fetch CLI args, initialize - # metrics (eg. New Relic) and so on in a way that - # helps prevent so much boilerplate code. - - db_client = db.init() - db_client.check_db_connection() - with db_client.get_session() as db_session: - script_context = ScriptContext(db_session) - yield script_context - logger.info("Script complete") diff --git a/app/api/services/users/create_user_csv.py b/app/api/services/users/create_user_csv.py index eb4e5781..761d288e 100644 --- a/app/api/services/users/create_user_csv.py +++ b/app/api/services/users/create_user_csv.py @@ -1,15 +1,11 @@ -# TODO move create_csv_script to a flask cli command so we don't need script_util.script_context_manager import csv import logging -import os from dataclasses import asdict, dataclass from smart_open import open as smart_open import api.adapters.db as db from api.db.models.user_models import User -from api.scripts.util.script_util import script_context_manager -from api.util.datetime_util import utcnow logger = logging.getLogger(__name__) From 1459046a6b920046966d3a4ee3d4ed9a3530d2fd Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Mon, 13 Feb 2023 13:06:10 -0800 Subject: [PATCH 29/51] Add comment to cli command --- app/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/Makefile b/app/Makefile index 28675dda..62f0017b 100644 --- a/app/Makefile +++ b/app/Makefile @@ -179,11 +179,11 @@ lint-security: # https://bandit.readthedocs.io/en/latest/index.html # CLI Commands ################################################## -cli: ## Run Flask app CLI command (Run make cli args="--help" to see list of CLI commands) +cli: ## Run Flask app CLI command (Run `make cli args="--help"` to see list of CLI commands) $(FLASK_CMD) $(args) -user-create-csv: - $(FLASK_CMD) user create-csv +cli-user-create-csv: ## Create a CSV of the useres in the database (Run `make cli-user-create-csv args="--help"` to see the command's options) + $(FLASK_CMD) user create-csv $(args) # Set init-db as pre-requisite since there seems to be a race condition # where the DB can't yet receive connections if it's starting from a From badb08249a10fe0e11128b710fe560c996cbb514 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Mon, 13 Feb 2023 13:21:23 -0800 Subject: [PATCH 30/51] Rename cli to cmd --- app/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Makefile b/app/Makefile index 62f0017b..41fea3cd 100644 --- a/app/Makefile +++ b/app/Makefile @@ -179,10 +179,10 @@ lint-security: # https://bandit.readthedocs.io/en/latest/index.html # CLI Commands ################################################## -cli: ## Run Flask app CLI command (Run `make cli args="--help"` to see list of CLI commands) +cmd: ## Run Flask app CLI command (Run `make cli args="--help"` to see list of CLI commands) $(FLASK_CMD) $(args) -cli-user-create-csv: ## Create a CSV of the useres in the database (Run `make cli-user-create-csv args="--help"` to see the command's options) +cmd-user-create-csv: ## Create a CSV of the useres in the database (Run `make cli-user-create-csv args="--help"` to see the command's options) $(FLASK_CMD) user create-csv $(args) # Set init-db as pre-requisite since there seems to be a race condition From 4cc0af45426ac2d520bdda656d1668877a673ed7 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Mon, 13 Feb 2023 13:31:53 -0800 Subject: [PATCH 31/51] Add help messages to cmd params --- app/api/api/users/user_commands.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py index 20682822..515bd4b4 100644 --- a/app/api/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -17,8 +17,16 @@ @user_blueprint.cli.command("create-csv", help="Create a CSV of all users and their roles") @flask_db.with_db_session -@click.option("--dir", default=".") -@click.option("--filename", default=None) +@click.option( + "--dir", + default=".", + help="Directory to save output file in. Can be an S3 path (e.g. 's3://bucketname/folder/') Defaults to current directory ('.').", +) +@click.option( + "--filename", + default=None, + help="Filename to save output file as. Defaults to '[timestamp]-user-roles.csv.", +) def create_csv(db_session: db.Session, dir: str, filename: Optional[str]) -> None: if filename is None: filename = utcnow().strftime("%Y-%m-%d-%H-%M-%S") + "-user-roles.csv" From 69f3848c2a4dcf2235fcc994e578fc80807dfed7 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Wed, 15 Feb 2023 10:47:35 -0800 Subject: [PATCH 32/51] Remove unused env var --- app/local.env | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/local.env b/app/local.env index 71961e47..63f135e9 100644 --- a/app/local.env +++ b/app/local.env @@ -74,8 +74,3 @@ AWS_SECRET_ACCESS_KEY=DO_NOT_SET_HERE #AWS_SESSION_TOKEN=DO_NOT_SET_HERE AWS_DEFAULT_REGION=us-east-1 - -############################ -# Example CSV Generation -############################ -USER_ROLE_CSV_OUTPUT_PATH = ./ From 0dc857f8d4b4ad36a7e4b05c0bc398b248e42b32 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 16 Feb 2023 13:22:00 -0800 Subject: [PATCH 33/51] Rename source directory from api to src --- app/Dockerfile | 2 +- app/Makefile | 22 +++++++++---------- app/local.env | 4 ++-- app/pyproject.toml | 14 ++++++------ app/{api => src}/__init__.py | 0 app/{api => src}/__main__.py | 10 ++++----- app/{api => src}/adapters/db/__init__.py | 4 ++-- app/{api => src}/adapters/db/client.py | 2 +- app/{api => src}/adapters/db/config.py | 2 +- app/{api => src}/adapters/db/flask_db.py | 16 +++++++------- app/{api => src}/api/__init__.py | 0 app/{api => src}/api/healthcheck.py | 6 ++--- app/{api => src}/api/response.py | 4 ++-- app/{api => src}/api/schemas/__init__.py | 0 .../api/schemas/request_schema.py | 0 .../api/schemas/response_schema.py | 2 +- app/{api => src}/api/users/__init__.py | 6 ++--- app/{api => src}/api/users/user_blueprint.py | 0 app/{api => src}/api/users/user_commands.py | 10 ++++----- app/{api => src}/api/users/user_routes.py | 18 +++++++-------- app/{api => src}/api/users/user_schemas.py | 4 ++-- app/{api => src}/app.py | 18 +++++++-------- app/{api => src}/app_config.py | 2 +- app/{api => src}/auth/__init__.py | 0 app/{api => src}/auth/api_key_auth.py | 0 app/{api => src}/db/__init__.py | 0 app/{api => src}/db/migrations/__init__.py | 0 app/{api => src}/db/migrations/alembic.ini | 2 +- app/{api => src}/db/migrations/env.py | 14 ++++++------ app/{api => src}/db/migrations/run.py | 0 app/{api => src}/db/migrations/script.py.mako | 0 .../2022_12_16_create_user_and_role_tables.py | 0 app/{api => src}/db/models/__init__.py | 0 app/{api => src}/db/models/base.py | 2 +- app/{api => src}/db/models/user_models.py | 2 +- app/{api => src}/logging/__init__.py | 10 ++++----- app/{api => src}/logging/audit.py | 4 ++-- app/{api => src}/logging/config.py | 10 ++++----- app/{api => src}/logging/decodelog.py | 2 +- app/{api => src}/logging/flask_logger.py | 8 +++---- app/{api => src}/logging/formatters.py | 2 +- app/{api => src}/logging/pii.py | 4 ++-- app/{api => src}/services/users/__init__.py | 0 .../services/users/create_user.py | 6 ++--- .../services/users/create_user_csv.py | 4 ++-- app/{api => src}/services/users/get_user.py | 4 ++-- app/{api => src}/services/users/patch_user.py | 6 ++--- app/{api => src}/util/__init__.py | 0 app/{api => src}/util/collections/__init__.py | 0 app/{api => src}/util/collections/dict.py | 0 app/{api => src}/util/datetime_util.py | 0 app/{api => src}/util/env_config.py | 4 ++-- app/{api => src}/util/file_util.py | 0 app/{api => src}/util/local.py | 0 app/{api => src}/util/string_utils.py | 0 app/tests/conftest.py | 10 ++++----- app/tests/lib/db_testing.py | 4 ++-- app/tests/{api => src}/__init__.py | 0 app/tests/{api => src}/adapters/__init__.py | 0 app/tests/{api => src}/adapters/db/test_db.py | 6 ++--- .../{api => src}/adapters/db/test_flask_db.py | 4 ++-- .../{api => src}/auth/test_api_key_auth.py | 2 +- app/tests/{api => src}/db/__init__.py | 0 app/tests/{api => src}/db/models/__init__.py | 0 app/tests/{api => src}/db/models/factories.py | 8 +++---- .../{api => src}/db/models/test_factories.py | 4 ++-- app/tests/{api => src}/db/test_migrations.py | 2 +- app/tests/{api => src}/logging/__init__.py | 0 app/tests/{api => src}/logging/test_audit.py | 4 ++-- .../{api => src}/logging/test_flask_logger.py | 6 ++--- .../{api => src}/logging/test_formatters.py | 2 +- .../{api => src}/logging/test_logging.py | 6 ++--- app/tests/{api => src}/logging/test_pii.py | 2 +- app/tests/{api => src}/route/__init__.py | 0 .../{api => src}/route/test_healthcheck.py | 2 +- .../{api => src}/route/test_user_route.py | 2 +- app/tests/{api => src}/scripts/__init__.py | 0 .../scripts/test_create_user_csv.py | 12 +++++----- .../scripts/test_create_user_csv_expected.csv | 0 app/tests/{api => src}/util/__init__.py | 0 .../{api => src}/util/collections/__init__.py | 0 .../util/collections/test_dict.py | 4 ++-- .../{api => src}/util/parametrize_utils.py | 0 .../{api => src}/util/test_datetime_util.py | 2 +- docs/app/README.md | 4 ++-- docs/app/api-details.md | 4 ++-- .../database/database-access-management.md | 6 ++--- docs/app/database/database-migrations.md | 2 +- docs/app/database/database-testing.md | 2 +- docs/app/formatting-and-linting.md | 2 +- .../logging-configuration.md | 10 ++++----- docs/app/writing-tests.md | 6 ++--- 92 files changed, 174 insertions(+), 174 deletions(-) rename app/{api => src}/__init__.py (100%) rename app/{api => src}/__main__.py (88%) rename app/{api => src}/adapters/db/__init__.py (89%) rename app/{api => src}/adapters/db/client.py (99%) rename app/{api => src}/adapters/db/config.py (95%) rename app/{api => src}/adapters/db/flask_db.py (88%) rename app/{api => src}/api/__init__.py (100%) rename app/{api => src}/api/healthcheck.py (89%) rename app/{api => src}/api/response.py (95%) rename app/{api => src}/api/schemas/__init__.py (100%) rename app/{api => src}/api/schemas/request_schema.py (100%) rename app/{api => src}/api/schemas/response_schema.py (95%) rename app/{api => src}/api/users/__init__.py (51%) rename app/{api => src}/api/users/user_blueprint.py (100%) rename app/{api => src}/api/users/user_commands.py (80%) rename app/{api => src}/api/users/user_routes.py (83%) rename app/{api => src}/api/users/user_schemas.py (94%) rename app/{api => src}/app.py (83%) rename app/{api => src}/app_config.py (89%) rename app/{api => src}/auth/__init__.py (100%) rename app/{api => src}/auth/api_key_auth.py (100%) rename app/{api => src}/db/__init__.py (100%) rename app/{api => src}/db/migrations/__init__.py (100%) rename app/{api => src}/db/migrations/alembic.ini (97%) rename app/{api => src}/db/migrations/env.py (89%) rename app/{api => src}/db/migrations/run.py (100%) rename app/{api => src}/db/migrations/script.py.mako (100%) rename app/{api => src}/db/migrations/versions/2022_12_16_create_user_and_role_tables.py (100%) rename app/{api => src}/db/models/__init__.py (100%) rename app/{api => src}/db/models/base.py (98%) rename app/{api => src}/db/models/user_models.py (96%) rename app/{api => src}/logging/__init__.py (91%) rename app/{api => src}/logging/audit.py (97%) rename app/{api => src}/logging/config.py (93%) rename app/{api => src}/logging/decodelog.py (99%) rename app/{api => src}/logging/flask_logger.py (95%) rename app/{api => src}/logging/formatters.py (97%) rename app/{api => src}/logging/pii.py (97%) rename app/{api => src}/services/users/__init__.py (100%) rename app/{api => src}/services/users/create_user.py (90%) rename app/{api => src}/services/users/create_user_csv.py (96%) rename app/{api => src}/services/users/get_user.py (91%) rename app/{api => src}/services/users/patch_user.py (94%) rename app/{api => src}/util/__init__.py (100%) rename app/{api => src}/util/collections/__init__.py (100%) rename app/{api => src}/util/collections/dict.py (100%) rename app/{api => src}/util/datetime_util.py (100%) rename app/{api => src}/util/env_config.py (78%) rename app/{api => src}/util/file_util.py (100%) rename app/{api => src}/util/local.py (100%) rename app/{api => src}/util/string_utils.py (100%) rename app/tests/{api => src}/__init__.py (100%) rename app/tests/{api => src}/adapters/__init__.py (100%) rename app/tests/{api => src}/adapters/db/test_db.py (96%) rename app/tests/{api => src}/adapters/db/test_flask_db.py (93%) rename app/tests/{api => src}/auth/test_api_key_auth.py (93%) rename app/tests/{api => src}/db/__init__.py (100%) rename app/tests/{api => src}/db/models/__init__.py (100%) rename app/tests/{api => src}/db/models/factories.py (93%) rename app/tests/{api => src}/db/models/test_factories.py (96%) rename app/tests/{api => src}/db/test_migrations.py (96%) rename app/tests/{api => src}/logging/__init__.py (100%) rename app/tests/{api => src}/logging/test_audit.py (99%) rename app/tests/{api => src}/logging/test_flask_logger.py (95%) rename app/tests/{api => src}/logging/test_formatters.py (97%) rename app/tests/{api => src}/logging/test_logging.py (95%) rename app/tests/{api => src}/logging/test_pii.py (96%) rename app/tests/{api => src}/route/__init__.py (100%) rename app/tests/{api => src}/route/test_healthcheck.py (94%) rename app/tests/{api => src}/route/test_user_route.py (99%) rename app/tests/{api => src}/scripts/__init__.py (100%) rename app/tests/{api => src}/scripts/test_create_user_csv.py (92%) rename app/tests/{api => src}/scripts/test_create_user_csv_expected.csv (100%) rename app/tests/{api => src}/util/__init__.py (100%) rename app/tests/{api => src}/util/collections/__init__.py (100%) rename app/tests/{api => src}/util/collections/test_dict.py (92%) rename app/tests/{api => src}/util/parametrize_utils.py (100%) rename app/tests/{api => src}/util/test_datetime_util.py (97%) diff --git a/app/Dockerfile b/app/Dockerfile index 6b944494..9ec44fbb 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -34,4 +34,4 @@ ENV HOST=0.0.0.0 # https://python-poetry.org/docs/basic-usage/#installing-dependencies RUN poetry install # Run the application. -CMD ["poetry", "run", "python", "-m", "api"] +CMD ["poetry", "run", "python", "-m", "src"] diff --git a/app/Makefile b/app/Makefile index 41fea3cd..244c3c99 100644 --- a/app/Makefile +++ b/app/Makefile @@ -10,15 +10,15 @@ APP_NAME := main-app # Note that you can also change the LOG_FORMAT env var to switch # between JSON & human readable format. This is left in place # in the event JSON is output from a process we don't log. -DECODE_LOG := 2>&1 | python3 -u api/logging/util/decodelog.py +DECODE_LOG := 2>&1 | python3 -u src/logging/util/decodelog.py # A few commands need adjustments if they're run in CI, specify those here # TODO - when CI gets hooked up, actually test this. ifdef CI DOCKER_EXEC_ARGS := -T -e CI -e PYTEST_ADDOPTS="--color=yes" - FLAKE8_FORMAT := '::warning file=api/%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + FLAKE8_FORMAT := '::warning file=src/%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s' MYPY_FLAGS := --no-pretty - MYPY_POSTPROC := | perl -pe "s/^(.+):(\d+):(\d+): error: (.*)/::warning file=api\/\1,line=\2,col=\3::\4/" + MYPY_POSTPROC := | perl -pe "s/^(.+):(\d+):(\d+): error: (.*)/::warning file=src\/\1,line=\2,col=\3::\4/" else FLAKE8_FORMAT := default endif @@ -92,7 +92,7 @@ db-recreate: clean-docker-volumes init-db # DB Migrations ######################### -alembic_config := ./api/db/migrations/alembic.ini +alembic_config := ./src/db/migrations/alembic.ini alembic_cmd := $(PY_RUN_CMD) alembic --config $(alembic_config) db-upgrade: ## Apply pending migrations to db @@ -139,7 +139,7 @@ test-watch: $(PY_RUN_CMD) pytest-watch --clear $(args) test-coverage: - $(PY_RUN_CMD) coverage run --branch --source=api -m pytest -m "not audit" $(args) + $(PY_RUN_CMD) coverage run --branch --source=src -m pytest -m "not audit" $(args) $(PY_RUN_CMD) coverage report test-coverage-report: ## Open HTML test coverage report @@ -151,22 +151,22 @@ test-coverage-report: ## Open HTML test coverage report ################################################## format: - $(PY_RUN_CMD) isort --atomic api tests - $(PY_RUN_CMD) black api tests + $(PY_RUN_CMD) isort --atomic src tests + $(PY_RUN_CMD) black src tests format-check: - $(PY_RUN_CMD) isort --atomic --check-only api tests - $(PY_RUN_CMD) black --check api tests + $(PY_RUN_CMD) isort --atomic --check-only src tests + $(PY_RUN_CMD) black --check src tests lint: lint-py lint-py: lint-flake lint-mypy lint-poetry-version lint-flake: - $(PY_RUN_CMD) flake8 --format=$(FLAKE8_FORMAT) api tests + $(PY_RUN_CMD) flake8 --format=$(FLAKE8_FORMAT) src tests lint-mypy: - $(PY_RUN_CMD) mypy --show-error-codes $(MYPY_FLAGS) api $(MYPY_POSTPROC) + $(PY_RUN_CMD) mypy --show-error-codes $(MYPY_FLAGS) src $(MYPY_POSTPROC) lint-poetry-version: ## Check poetry version grep --quiet 'lock-version = "1.1"' poetry.lock diff --git a/app/local.env b/app/local.env index 63f135e9..04c30a3a 100644 --- a/app/local.env +++ b/app/local.env @@ -1,6 +1,6 @@ # Local environment variables # Used by docker-compose and it can be loaded -# by calling load_local_env_vars() from app/api/util/local.py +# by calling load_local_env_vars() from app/src/util/local.py ENVIRONMENT=local PORT=8080 @@ -15,7 +15,7 @@ PYTHONPATH=/app/ # commands that can run in or out # of the Docker container - defaults to outside -FLASK_APP=api.app:create_app +FLASK_APP=src.app:create_app ############################ # Logging diff --git a/app/pyproject.toml b/app/pyproject.toml index eb85e2cd..7e4af099 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -2,7 +2,7 @@ name = "template-application-flask" version = "0.1.0" description = "A template flask API for building ontop of" -packages = [{ include = "api" }] +packages = [{ include = "src" }] authors = ["Nava Engineering "] [tool.poetry.dependencies] @@ -44,9 +44,9 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -db-migrate-up = "api.db.migrations.run:up" -db-migrate-down = "api.db.migrations.run:down" -db-migrate-down-all = "api.db.migrations.run:downall" +db-migrate-up = "src.db.migrations.run:up" +db-migrate-down = "src.db.migrations.run:down" +db-migrate-down-all = "src.db.migrations.run:downall" [tool.black] line-length = 100 @@ -85,14 +85,14 @@ plugins = ["sqlalchemy.ext.mypy.plugin"] [tool.bandit] # Ignore audit logging test file since test audit logging requires a lot of operations that trigger bandit warnings -exclude_dirs = ["./tests/api/logging/test_audit.py"] +exclude_dirs = ["./tests/src/logging/test_audit.py"] [[tool.mypy.overrides]] # Migrations are generated without "-> None" # for the returns. Rather than require manually # fixing this for every migration generated, # disable the check for that folder. -module = "api.db.migrations.versions.*" +module = "src.db.migrations.versions.*" disallow_untyped_defs = false [tool.pytest.ini_options] @@ -108,5 +108,5 @@ markers = [ "audit: mark a test as a security audit log test, to be run isolated from other tests"] [tool.coverage.run] -omit = ["api/db/migrations/*.py"] +omit = ["src/db/migrations/*.py"] diff --git a/app/api/__init__.py b/app/src/__init__.py similarity index 100% rename from app/api/__init__.py rename to app/src/__init__.py diff --git a/app/api/__main__.py b/app/src/__main__.py similarity index 88% rename from app/api/__main__.py rename to app/src/__main__.py index aab01a51..725a6f2d 100644 --- a/app/api/__main__.py +++ b/app/src/__main__.py @@ -7,10 +7,10 @@ import logging -import api.app -import api.logging -from api.app_config import AppConfig -from api.util.local import load_local_env_vars +import src.app +import src.logging +from src.app_config import AppConfig +from src.util.local import load_local_env_vars logger = logging.getLogger(__package__) @@ -19,7 +19,7 @@ def main() -> None: load_local_env_vars() app_config = AppConfig() - app = api.app.create_app() + app = src.app.create_app() environment = app_config.environment diff --git a/app/api/adapters/db/__init__.py b/app/src/adapters/db/__init__.py similarity index 89% rename from app/api/adapters/db/__init__.py rename to app/src/adapters/db/__init__.py index 9f415607..ddb68e4e 100644 --- a/app/api/adapters/db/__init__.py +++ b/app/src/adapters/db/__init__.py @@ -7,7 +7,7 @@ To use this module with Flask, use the flask_db module. Usage: - import api.adapters.db as db + import src.adapters.db as db db_client = db.init() @@ -23,7 +23,7 @@ """ # Re-export for convenience -from api.adapters.db.client import Connection, DBClient, Session, init +from src.adapters.db.client import Connection, DBClient, Session, init # Do not import flask_db here, because this module is not dependent on any specific framework. # Code can choose to use this module on its own or with the flask_db module depending on needs. diff --git a/app/api/adapters/db/client.py b/app/src/adapters/db/client.py similarity index 99% rename from app/api/adapters/db/client.py rename to app/src/adapters/db/client.py index 36f38eee..3f55e357 100644 --- a/app/api/adapters/db/client.py +++ b/app/src/adapters/db/client.py @@ -17,7 +17,7 @@ import sqlalchemy.pool as pool from sqlalchemy.orm import session -from api.adapters.db.config import DbConfig, get_db_config +from src.adapters.db.config import DbConfig, get_db_config # Re-export the Connection type that is returned by the get_connection() method # to be used for type hints. diff --git a/app/api/adapters/db/config.py b/app/src/adapters/db/config.py similarity index 95% rename from app/api/adapters/db/config.py rename to app/src/adapters/db/config.py index 67475f65..dff1d91f 100644 --- a/app/api/adapters/db/config.py +++ b/app/src/adapters/db/config.py @@ -3,7 +3,7 @@ from pydantic import Field -from api.util.env_config import PydanticBaseEnvConfig +from src.util.env_config import PydanticBaseEnvConfig logger = logging.getLogger(__name__) diff --git a/app/api/adapters/db/flask_db.py b/app/src/adapters/db/flask_db.py similarity index 88% rename from app/api/adapters/db/flask_db.py rename to app/src/adapters/db/flask_db.py index 04f486c3..24960958 100644 --- a/app/api/adapters/db/flask_db.py +++ b/app/src/adapters/db/flask_db.py @@ -5,8 +5,8 @@ of a Flask app and an instance of a DBClient. Example: - import api.adapters.db as db - import api.adapters.db.flask_db as flask_db + import src.adapters.db as db + import src.adapters.db.flask_db as flask_db db_client = db.init() app = APIFlask(__name__) @@ -16,8 +16,8 @@ new database session that lasts for the duration of the request. Example: - import api.adapters.db as db - import api.adapters.db.flask_db as flask_db + import src.adapters.db as db + import src.adapters.db.flask_db as flask_db @app.route("/health") @flask_db.with_db_session @@ -31,7 +31,7 @@ def health(db_session: db.Session): Example: from flask import current_app - import api.adapters.db.flask_db as flask_db + import src.adapters.db.flask_db as flask_db @app.route("/health") def health(): @@ -43,8 +43,8 @@ def health(): from flask import Flask, current_app -import api.adapters.db as db -from api.adapters.db.client import DBClient +import src.adapters.db as db +from src.adapters.db.client import DBClient _FLASK_EXTENSION_KEY = "db" @@ -67,7 +67,7 @@ def get_db(app: Flask) -> DBClient: Example: from flask import current_app - import api.adapters.db.flask_db as flask_db + import src.adapters.db.flask_db as flask_db @app.route("/health") def health(): diff --git a/app/api/api/__init__.py b/app/src/api/__init__.py similarity index 100% rename from app/api/api/__init__.py rename to app/src/api/__init__.py diff --git a/app/api/api/healthcheck.py b/app/src/api/healthcheck.py similarity index 89% rename from app/api/api/healthcheck.py rename to app/src/api/healthcheck.py index 4df442b2..4dc5782b 100644 --- a/app/api/api/healthcheck.py +++ b/app/src/api/healthcheck.py @@ -6,9 +6,9 @@ from sqlalchemy import text from werkzeug.exceptions import ServiceUnavailable -import api.adapters.db.flask_db as flask_db -from api.api import response -from api.api.schemas import request_schema +import src.adapters.db.flask_db as flask_db +from src.api import response +from src.api.schemas import request_schema logger = logging.getLogger(__name__) diff --git a/app/api/api/response.py b/app/src/api/response.py similarity index 95% rename from app/api/api/response.py rename to app/src/api/response.py index 65f6e0bb..70e2ae8f 100644 --- a/app/api/api/response.py +++ b/app/src/api/response.py @@ -1,8 +1,8 @@ import dataclasses from typing import Optional -from api.api.schemas import response_schema -from api.db.models.base import Base +from src.api.schemas import response_schema +from src.db.models.base import Base @dataclasses.dataclass diff --git a/app/api/api/schemas/__init__.py b/app/src/api/schemas/__init__.py similarity index 100% rename from app/api/api/schemas/__init__.py rename to app/src/api/schemas/__init__.py diff --git a/app/api/api/schemas/request_schema.py b/app/src/api/schemas/request_schema.py similarity index 100% rename from app/api/api/schemas/request_schema.py rename to app/src/api/schemas/request_schema.py diff --git a/app/api/api/schemas/response_schema.py b/app/src/api/schemas/response_schema.py similarity index 95% rename from app/api/api/schemas/response_schema.py rename to app/src/api/schemas/response_schema.py index 9ae6f6ac..5d89e85e 100644 --- a/app/api/api/schemas/response_schema.py +++ b/app/src/api/schemas/response_schema.py @@ -1,6 +1,6 @@ from apiflask import fields -from api.api.schemas import request_schema +from src.api.schemas import request_schema class ValidationErrorSchema(request_schema.OrderedSchema): diff --git a/app/api/api/users/__init__.py b/app/src/api/users/__init__.py similarity index 51% rename from app/api/api/users/__init__.py rename to app/src/api/users/__init__.py index 1da0ac9f..8ddfd549 100644 --- a/app/api/api/users/__init__.py +++ b/app/src/api/users/__init__.py @@ -1,10 +1,10 @@ -from api.api.users.user_blueprint import user_blueprint +from src.api.users.user_blueprint import user_blueprint # import user_commands module to register the CLI commands on the user_blueprint -import api.api.users.user_commands # noqa: F401 E402 isort:skip +import src.api.users.user_commands # noqa: F401 E402 isort:skip # import user_commands module to register the API routes on the user_blueprint -import api.api.users.user_routes # noqa: F401 E402 isort:skip +import src.api.users.user_routes # noqa: F401 E402 isort:skip __all__ = ["user_blueprint"] diff --git a/app/api/api/users/user_blueprint.py b/app/src/api/users/user_blueprint.py similarity index 100% rename from app/api/api/users/user_blueprint.py rename to app/src/api/users/user_blueprint.py diff --git a/app/api/api/users/user_commands.py b/app/src/api/users/user_commands.py similarity index 80% rename from app/api/api/users/user_commands.py rename to app/src/api/users/user_commands.py index 515bd4b4..71f23f57 100644 --- a/app/api/api/users/user_commands.py +++ b/app/src/api/users/user_commands.py @@ -4,11 +4,11 @@ import click -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db -import api.services.users as user_service -from api.api.users.user_blueprint import user_blueprint -from api.util.datetime_util import utcnow +import src.adapters.db as db +import src.adapters.db.flask_db as flask_db +import src.services.users as user_service +from src.api.users.user_blueprint import user_blueprint +from src.util.datetime_util import utcnow logger = logging.getLogger(__name__) diff --git a/app/api/api/users/user_routes.py b/app/src/api/users/user_routes.py similarity index 83% rename from app/api/api/users/user_routes.py rename to app/src/api/users/user_routes.py index 5ef77eee..72163d72 100644 --- a/app/api/api/users/user_routes.py +++ b/app/src/api/users/user_routes.py @@ -1,15 +1,15 @@ import logging from typing import Any -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db -import api.api.response as response -import api.api.users.user_schemas as user_schemas -import api.services.users as user_service -import api.services.users as users -from api.api.users.user_blueprint import user_blueprint -from api.auth.api_key_auth import api_key_auth -from api.db.models.user_models import User +import src.adapters.db as db +import src.adapters.db.flask_db as flask_db +import src.api.response as response +import src.api.users.user_schemas as user_schemas +import src.services.users as user_service +import src.services.users as users +from src.api.users.user_blueprint import user_blueprint +from src.auth.api_key_auth import api_key_auth +from src.db.models.user_models import User logger = logging.getLogger(__name__) diff --git a/app/api/api/users/user_schemas.py b/app/src/api/users/user_schemas.py similarity index 94% rename from app/api/api/users/user_schemas.py rename to app/src/api/users/user_schemas.py index cc44c6b1..b0fd2c14 100644 --- a/app/api/api/users/user_schemas.py +++ b/app/src/api/users/user_schemas.py @@ -1,8 +1,8 @@ from apiflask import fields from marshmallow import fields as marshmallow_fields -from api.api.schemas import request_schema -from api.db.models import user_models +from src.api.schemas import request_schema +from src.db.models import user_models class RoleSchema(request_schema.OrderedSchema): diff --git a/app/api/app.py b/app/src/app.py similarity index 83% rename from app/api/app.py rename to app/src/app.py index 4c118c90..f1d72806 100644 --- a/app/api/app.py +++ b/app/src/app.py @@ -6,14 +6,14 @@ from flask import g from werkzeug.exceptions import Unauthorized -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db -import api.logging -import api.logging.flask_logger as flask_logger -from api.api.healthcheck import healthcheck_blueprint -from api.api.schemas import response_schema -from api.api.users import user_blueprint -from api.auth.api_key_auth import User, get_app_security_scheme +import src.adapters.db as db +import src.adapters.db.flask_db as flask_db +import src.logging +import src.logging.flask_logger as flask_logger +from src.api.healthcheck import healthcheck_blueprint +from src.api.schemas import response_schema +from src.api.users import user_blueprint +from src.auth.api_key_auth import User, get_app_security_scheme logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def create_app() -> APIFlask: app = APIFlask(__name__) - root_logger = api.logging.init(__package__) + root_logger = src.logging.init(__package__) flask_logger.init_app(root_logger, app) db_client = db.init() diff --git a/app/api/app_config.py b/app/src/app_config.py similarity index 89% rename from app/api/app_config.py rename to app/src/app_config.py index e97be25f..44159955 100644 --- a/app/api/app_config.py +++ b/app/src/app_config.py @@ -1,4 +1,4 @@ -from api.util.env_config import PydanticBaseEnvConfig +from src.util.env_config import PydanticBaseEnvConfig class AppConfig(PydanticBaseEnvConfig): diff --git a/app/api/auth/__init__.py b/app/src/auth/__init__.py similarity index 100% rename from app/api/auth/__init__.py rename to app/src/auth/__init__.py diff --git a/app/api/auth/api_key_auth.py b/app/src/auth/api_key_auth.py similarity index 100% rename from app/api/auth/api_key_auth.py rename to app/src/auth/api_key_auth.py diff --git a/app/api/db/__init__.py b/app/src/db/__init__.py similarity index 100% rename from app/api/db/__init__.py rename to app/src/db/__init__.py diff --git a/app/api/db/migrations/__init__.py b/app/src/db/migrations/__init__.py similarity index 100% rename from app/api/db/migrations/__init__.py rename to app/src/db/migrations/__init__.py diff --git a/app/api/db/migrations/alembic.ini b/app/src/db/migrations/alembic.ini similarity index 97% rename from app/api/db/migrations/alembic.ini rename to app/src/db/migrations/alembic.ini index 8850aced..7f64607f 100644 --- a/app/api/db/migrations/alembic.ini +++ b/app/src/db/migrations/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = api/db/migrations +script_location = src/db/migrations file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(slug)s diff --git a/app/api/db/migrations/env.py b/app/src/db/migrations/env.py similarity index 89% rename from app/api/db/migrations/env.py rename to app/src/db/migrations/env.py index 50b105f1..e519705a 100644 --- a/app/api/db/migrations/env.py +++ b/app/src/db/migrations/env.py @@ -6,19 +6,19 @@ from alembic import context # Alembic cli seems to reset the path on load causing issues with local module imports. -# Workaround is to force set the path to the current run directory (top level api folder) +# Workaround is to force set the path to the current run directory (top level src folder) # See database migrations section in `./database/database-migrations.md` for details about running migrations. sys.path.insert(0, ".") # noqa: E402 # Load env vars before anything further -from api.util.local import load_local_env_vars # noqa: E402 isort:skip +from src.util.local import load_local_env_vars # noqa: E402 isort:skip load_local_env_vars() -from api.adapters.db.client import make_connection_uri # noqa: E402 isort:skip -from api.adapters.db.config import get_db_config # noqa: E402 isort:skip -from api.db.models import metadata # noqa: E402 isort:skip -import api.logging # noqa: E402 isort:skip +from src.adapters.db.client import make_connection_uri # noqa: E402 isort:skip +from src.adapters.db.config import get_db_config # noqa: E402 isort:skip +from src.db.models import metadata # noqa: E402 isort:skip +import src.logging # noqa: E402 isort:skip # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -27,7 +27,7 @@ logger = logging.getLogger("migrations") # Initialize logging -api.logging.init("migrations") +src.logging.init("migrations") if not config.get_main_option("sqlalchemy.url"): uri = make_connection_uri(get_db_config()) diff --git a/app/api/db/migrations/run.py b/app/src/db/migrations/run.py similarity index 100% rename from app/api/db/migrations/run.py rename to app/src/db/migrations/run.py diff --git a/app/api/db/migrations/script.py.mako b/app/src/db/migrations/script.py.mako similarity index 100% rename from app/api/db/migrations/script.py.mako rename to app/src/db/migrations/script.py.mako diff --git a/app/api/db/migrations/versions/2022_12_16_create_user_and_role_tables.py b/app/src/db/migrations/versions/2022_12_16_create_user_and_role_tables.py similarity index 100% rename from app/api/db/migrations/versions/2022_12_16_create_user_and_role_tables.py rename to app/src/db/migrations/versions/2022_12_16_create_user_and_role_tables.py diff --git a/app/api/db/models/__init__.py b/app/src/db/models/__init__.py similarity index 100% rename from app/api/db/models/__init__.py rename to app/src/db/models/__init__.py diff --git a/app/api/db/models/base.py b/app/src/db/models/base.py similarity index 98% rename from app/api/db/models/base.py rename to app/src/db/models/base.py index 111f688e..6cb4d9b8 100644 --- a/app/api/db/models/base.py +++ b/app/src/db/models/base.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import declarative_mixin from sqlalchemy.sql.functions import now as sqlnow -from api.util import datetime_util +from src.util import datetime_util # Override the default naming of constraints # to use suffixes instead: diff --git a/app/api/db/models/user_models.py b/app/src/db/models/user_models.py similarity index 96% rename from app/api/db/models/user_models.py rename to app/src/db/models/user_models.py index 47341160..6631f3ee 100644 --- a/app/api/db/models/user_models.py +++ b/app/src/db/models/user_models.py @@ -8,7 +8,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Mapped, relationship -from api.db.models.base import Base, IdMixin, TimestampMixin +from src.db.models.base import Base, IdMixin, TimestampMixin logger = logging.getLogger(__name__) diff --git a/app/api/logging/__init__.py b/app/src/logging/__init__.py similarity index 91% rename from app/api/logging/__init__.py rename to app/src/logging/__init__.py index f1178bc6..463b4eec 100644 --- a/app/api/logging/__init__.py +++ b/app/src/logging/__init__.py @@ -3,15 +3,15 @@ There are two formatters for the log messages: human-readable and JSON. The formatter that is used is determined by the environment variable LOG_FORMAT. If the environment variable is not set, the JSON formatter -is used by default. See api.logging.formatters for more information. +is used by default. See src.logging.formatters for more information. The logger also adds a PII mask filter to the root logger. See -api.logging.pii for more information. +src.logging.pii for more information. Usage: - import api.logging + import src.logging - api.logging.init("program name") + src.logging.init("program name") Once the module has been initialized, the standard logging module can be used to log messages: @@ -30,7 +30,7 @@ import sys from typing import Any, cast -import api.logging.config as config +import src.logging.config as config logger = logging.getLogger(__name__) _original_argv = tuple(sys.argv) diff --git a/app/api/logging/audit.py b/app/src/logging/audit.py similarity index 97% rename from app/api/logging/audit.py rename to app/src/logging/audit.py index a203992b..91a8f33b 100644 --- a/app/api/logging/audit.py +++ b/app/src/logging/audit.py @@ -10,7 +10,7 @@ import sys from typing import Any, Sequence -import api.util.collections +import src.util.collections logger = logging.getLogger(__name__) @@ -95,4 +95,4 @@ def log_audit_event(event_name: str, args: Sequence[Any], arg_names: Sequence[st logger.log(AUDIT, event_name, extra=extra) -audit_message_count = api.util.collections.LeastRecentlyUsedDict() +audit_message_count = src.util.collections.LeastRecentlyUsedDict() diff --git a/app/api/logging/config.py b/app/src/logging/config.py similarity index 93% rename from app/api/logging/config.py rename to app/src/logging/config.py index d58e02af..46ac7324 100644 --- a/app/api/logging/config.py +++ b/app/src/logging/config.py @@ -1,10 +1,10 @@ import logging import sys -import api.logging.audit -import api.logging.formatters as formatters -import api.logging.pii as pii -from api.util.env_config import PydanticBaseEnvConfig +import src.logging.audit +import src.logging.formatters as formatters +import src.logging.pii as pii +from src.util.env_config import PydanticBaseEnvConfig logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def configure_logging() -> logging.Logger: logging.root.setLevel(config.level) if config.enable_audit: - api.logging.audit.init() + src.logging.audit.init() # Configure loggers for third party packages logging.getLogger("alembic").setLevel(logging.INFO) diff --git a/app/api/logging/decodelog.py b/app/src/logging/decodelog.py similarity index 99% rename from app/api/logging/decodelog.py rename to app/src/logging/decodelog.py index 3949d49a..c8211f96 100644 --- a/app/api/logging/decodelog.py +++ b/app/src/logging/decodelog.py @@ -94,7 +94,7 @@ def colorize(text: str, color: str) -> str: def color_for_name(name: str) -> str: - if name.startswith("api"): + if name.startswith("src"): return GREEN elif name.startswith("sqlalchemy"): return ORANGE diff --git a/app/api/logging/flask_logger.py b/app/src/logging/flask_logger.py similarity index 95% rename from app/api/logging/flask_logger.py rename to app/src/logging/flask_logger.py index ea517916..73e09cd3 100644 --- a/app/api/logging/flask_logger.py +++ b/app/src/logging/flask_logger.py @@ -9,7 +9,7 @@ non-404 request. Usage: - import api.logging.flask_logger as flask_logger + import src.logging.flask_logger as flask_logger logger = logging.getLogger(__name__) app = create_app() @@ -35,7 +35,7 @@ def init_app(app_logger: logging.Logger, app: flask.Flask) -> None: Also configures the app to log every non-404 request using the given logger. Usage: - import api.logging.flask_logger as flask_logger + import src.logging.flask_logger as flask_logger logger = logging.getLogger(__name__) app = create_app() @@ -77,7 +77,7 @@ def _log_start_request() -> None: """Log the start of a request. This function handles the Flask's before_request event. - See https://tedboy.github.io/flask/interface_api.application_object.html#flask.Flask.before_request + See https://tedboy.github.io/flask/interface_src.application_object.html#flask.Flask.before_request Additional info about the request will be in the `extra` field added by `_add_request_context_info_to_log_record` @@ -89,7 +89,7 @@ def _log_end_request(response: flask.Response) -> flask.Response: """Log the end of a request. This function handles the Flask's after_request event. - See https://tedboy.github.io/flask/interface_api.application_object.html#flask.Flask.after_request + See https://tedboy.github.io/flask/interface_src.application_object.html#flask.Flask.after_request Additional info about the request will be in the `extra` field added by `_add_request_context_info_to_log_record` diff --git a/app/api/logging/formatters.py b/app/src/logging/formatters.py similarity index 97% rename from app/api/logging/formatters.py rename to app/src/logging/formatters.py index e93cbe32..abd1d00d 100644 --- a/app/api/logging/formatters.py +++ b/app/src/logging/formatters.py @@ -11,7 +11,7 @@ import logging from datetime import datetime -import api.logging.decodelog as decodelog +import src.logging.decodelog as decodelog class JsonFormatter(logging.Formatter): diff --git a/app/api/logging/pii.py b/app/src/logging/pii.py similarity index 97% rename from app/api/logging/pii.py rename to app/src/logging/pii.py index 67098618..8cce58f9 100644 --- a/app/api/logging/pii.py +++ b/app/src/logging/pii.py @@ -8,7 +8,7 @@ Example: import logging - import api.logging.pii as pii + import src.logging.pii as pii handler = logging.StreamHandler() handler.addFilter(pii.mask_pii) @@ -22,7 +22,7 @@ Example: import logging - import api.logging.pii as pii + import src.logging.pii as pii logger = logging.getLogger(__name__) logger.addFilter(pii.mask_pii) diff --git a/app/api/services/users/__init__.py b/app/src/services/users/__init__.py similarity index 100% rename from app/api/services/users/__init__.py rename to app/src/services/users/__init__.py diff --git a/app/api/services/users/create_user.py b/app/src/services/users/create_user.py similarity index 90% rename from app/api/services/users/create_user.py rename to app/src/services/users/create_user.py index 096f9621..9721594a 100644 --- a/app/api/services/users/create_user.py +++ b/app/src/services/users/create_user.py @@ -1,9 +1,9 @@ from datetime import date from typing import TypedDict -from api.adapters.db import Session -from api.db.models import user_models -from api.db.models.user_models import Role, User +from src.adapters.db import Session +from src.db.models import user_models +from src.db.models.user_models import Role, User class RoleParams(TypedDict): diff --git a/app/api/services/users/create_user_csv.py b/app/src/services/users/create_user_csv.py similarity index 96% rename from app/api/services/users/create_user_csv.py rename to app/src/services/users/create_user_csv.py index 761d288e..959012f6 100644 --- a/app/api/services/users/create_user_csv.py +++ b/app/src/services/users/create_user_csv.py @@ -4,8 +4,8 @@ from smart_open import open as smart_open -import api.adapters.db as db -from api.db.models.user_models import User +import src.adapters.db as db +from src.db.models.user_models import User logger = logging.getLogger(__name__) diff --git a/app/api/services/users/get_user.py b/app/src/services/users/get_user.py similarity index 91% rename from app/api/services/users/get_user.py rename to app/src/services/users/get_user.py index 2947ad69..4af4c61c 100644 --- a/app/api/services/users/get_user.py +++ b/app/src/services/users/get_user.py @@ -1,8 +1,8 @@ import apiflask from sqlalchemy import orm -from api.adapters.db import Session -from api.db.models.user_models import User +from src.adapters.db import Session +from src.db.models.user_models import User # TODO: separate controller and service concerns diff --git a/app/api/services/users/patch_user.py b/app/src/services/users/patch_user.py similarity index 94% rename from app/api/services/users/patch_user.py rename to app/src/services/users/patch_user.py index c71653e4..bb23a22d 100644 --- a/app/api/services/users/patch_user.py +++ b/app/src/services/users/patch_user.py @@ -6,9 +6,9 @@ import apiflask from sqlalchemy import orm -from api.adapters.db import Session -from api.db.models.user_models import Role, User -from api.services.users.create_user import RoleParams +from src.adapters.db import Session +from src.db.models.user_models import Role, User +from src.services.users.create_user import RoleParams class PatchUserParams(TypedDict, total=False): diff --git a/app/api/util/__init__.py b/app/src/util/__init__.py similarity index 100% rename from app/api/util/__init__.py rename to app/src/util/__init__.py diff --git a/app/api/util/collections/__init__.py b/app/src/util/collections/__init__.py similarity index 100% rename from app/api/util/collections/__init__.py rename to app/src/util/collections/__init__.py diff --git a/app/api/util/collections/dict.py b/app/src/util/collections/dict.py similarity index 100% rename from app/api/util/collections/dict.py rename to app/src/util/collections/dict.py diff --git a/app/api/util/datetime_util.py b/app/src/util/datetime_util.py similarity index 100% rename from app/api/util/datetime_util.py rename to app/src/util/datetime_util.py diff --git a/app/api/util/env_config.py b/app/src/util/env_config.py similarity index 78% rename from app/api/util/env_config.py rename to app/src/util/env_config.py index c556c465..cd9c4dc3 100644 --- a/app/api/util/env_config.py +++ b/app/src/util/env_config.py @@ -2,10 +2,10 @@ from pydantic import BaseSettings -import api +import src env_file = os.path.join( - os.path.dirname(os.path.dirname(api.__file__)), + os.path.dirname(os.path.dirname(src.__file__)), "config", "%s.env" % os.getenv("ENVIRONMENT", "local"), ) diff --git a/app/api/util/file_util.py b/app/src/util/file_util.py similarity index 100% rename from app/api/util/file_util.py rename to app/src/util/file_util.py diff --git a/app/api/util/local.py b/app/src/util/local.py similarity index 100% rename from app/api/util/local.py rename to app/src/util/local.py diff --git a/app/api/util/string_utils.py b/app/src/util/string_utils.py similarity index 100% rename from app/api/util/string_utils.py rename to app/src/util/string_utils.py diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 8bd26567..66a8e1e7 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -8,11 +8,11 @@ import pytest import sqlalchemy -import api.adapters.db as db -import api.app as app_entry -import tests.api.db.models.factories as factories -from api.db import models -from api.util.local import load_local_env_vars +import src.adapters.db as db +import src.app as app_entry +import tests.src.db.models.factories as factories +from src.db import models +from src.util.local import load_local_env_vars from tests.lib import db_testing logger = logging.getLogger(__name__) diff --git a/app/tests/lib/db_testing.py b/app/tests/lib/db_testing.py index 911d113e..05a8fef5 100644 --- a/app/tests/lib/db_testing.py +++ b/app/tests/lib/db_testing.py @@ -3,8 +3,8 @@ import logging import uuid -import api.adapters.db as db -from api.adapters.db.config import get_db_config +import src.adapters.db as db +from src.adapters.db.config import get_db_config logger = logging.getLogger(__name__) diff --git a/app/tests/api/__init__.py b/app/tests/src/__init__.py similarity index 100% rename from app/tests/api/__init__.py rename to app/tests/src/__init__.py diff --git a/app/tests/api/adapters/__init__.py b/app/tests/src/adapters/__init__.py similarity index 100% rename from app/tests/api/adapters/__init__.py rename to app/tests/src/adapters/__init__.py diff --git a/app/tests/api/adapters/db/test_db.py b/app/tests/src/adapters/db/test_db.py similarity index 96% rename from app/tests/api/adapters/db/test_db.py rename to app/tests/src/adapters/db/test_db.py index de68e044..77f2c8f8 100644 --- a/app/tests/api/adapters/db/test_db.py +++ b/app/tests/src/adapters/db/test_db.py @@ -4,9 +4,9 @@ import pytest from sqlalchemy import text -import api.adapters.db as db -from api.adapters.db.client import get_connection_parameters, make_connection_uri, verify_ssl -from api.adapters.db.config import DbConfig, get_db_config +import src.adapters.db as db +from src.adapters.db.client import get_connection_parameters, make_connection_uri, verify_ssl +from src.adapters.db.config import DbConfig, get_db_config class DummyConnectionInfo: diff --git a/app/tests/api/adapters/db/test_flask_db.py b/app/tests/src/adapters/db/test_flask_db.py similarity index 93% rename from app/tests/api/adapters/db/test_flask_db.py rename to app/tests/src/adapters/db/test_flask_db.py index dfca9964..03c5ccd9 100644 --- a/app/tests/api/adapters/db/test_flask_db.py +++ b/app/tests/src/adapters/db/test_flask_db.py @@ -2,8 +2,8 @@ from flask import Flask, current_app from sqlalchemy import text -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db +import src.adapters.db as db +import src.adapters.db.flask_db as flask_db # Define an isolated example Flask app fixture specific to this test module diff --git a/app/tests/api/auth/test_api_key_auth.py b/app/tests/src/auth/test_api_key_auth.py similarity index 93% rename from app/tests/api/auth/test_api_key_auth.py rename to app/tests/src/auth/test_api_key_auth.py index c3434d7b..152cc63f 100644 --- a/app/tests/api/auth/test_api_key_auth.py +++ b/app/tests/src/auth/test_api_key_auth.py @@ -2,7 +2,7 @@ from apiflask import HTTPError from flask import g -from api.auth.api_key_auth import API_AUTH_USER, verify_token +from src.auth.api_key_auth import API_AUTH_USER, verify_token def test_verify_token_success(app, api_auth_token): diff --git a/app/tests/api/db/__init__.py b/app/tests/src/db/__init__.py similarity index 100% rename from app/tests/api/db/__init__.py rename to app/tests/src/db/__init__.py diff --git a/app/tests/api/db/models/__init__.py b/app/tests/src/db/models/__init__.py similarity index 100% rename from app/tests/api/db/models/__init__.py rename to app/tests/src/db/models/__init__.py diff --git a/app/tests/api/db/models/factories.py b/app/tests/src/db/models/factories.py similarity index 93% rename from app/tests/api/db/models/factories.py rename to app/tests/src/db/models/factories.py index c822f680..7edf8118 100644 --- a/app/tests/api/db/models/factories.py +++ b/app/tests/src/db/models/factories.py @@ -15,9 +15,9 @@ import faker from sqlalchemy.orm import scoped_session -import api.adapters.db as db -import api.db.models.user_models as user_models -import api.util.datetime_util as datetime_util +import src.adapters.db as db +import src.db.models.user_models as user_models +import src.util.datetime_util as datetime_util _db_session: Optional[db.Session] = None @@ -67,7 +67,7 @@ class Meta: model = user_models.Role user_id = factory.LazyAttribute(lambda u: u.user.id) - user = factory.SubFactory("tests.api.db.models.factories.UserFactory", roles=[]) + user = factory.SubFactory("tests.src.db.models.factories.UserFactory", roles=[]) type = factory.Iterator([r.value for r in user_models.RoleType]) diff --git a/app/tests/api/db/models/test_factories.py b/app/tests/src/db/models/test_factories.py similarity index 96% rename from app/tests/api/db/models/test_factories.py rename to app/tests/src/db/models/test_factories.py index 79807224..b3370a3f 100644 --- a/app/tests/api/db/models/test_factories.py +++ b/app/tests/src/db/models/test_factories.py @@ -2,8 +2,8 @@ import pytest -from api.db.models.user_models import User -from tests.api.db.models.factories import RoleFactory, UserFactory +from src.db.models.user_models import User +from tests.src.db.models.factories import RoleFactory, UserFactory user_params = { "first_name": "Alvin", diff --git a/app/tests/api/db/test_migrations.py b/app/tests/src/db/test_migrations.py similarity index 96% rename from app/tests/api/db/test_migrations.py rename to app/tests/src/db/test_migrations.py index 0f286631..9a54ae03 100644 --- a/app/tests/api/db/test_migrations.py +++ b/app/tests/src/db/test_migrations.py @@ -6,7 +6,7 @@ from alembic.script.revision import MultipleHeads from alembic.util.exc import CommandError -from api.db.migrations.run import alembic_cfg +from src.db.migrations.run import alembic_cfg def test_only_single_head_revision_in_migrations(): diff --git a/app/tests/api/logging/__init__.py b/app/tests/src/logging/__init__.py similarity index 100% rename from app/tests/api/logging/__init__.py rename to app/tests/src/logging/__init__.py diff --git a/app/tests/api/logging/test_audit.py b/app/tests/src/logging/test_audit.py similarity index 99% rename from app/tests/api/logging/test_audit.py rename to app/tests/src/logging/test_audit.py index 874c9a9b..70213b5e 100644 --- a/app/tests/api/logging/test_audit.py +++ b/app/tests/src/logging/test_audit.py @@ -1,5 +1,5 @@ # -# Tests for api.logging.audit. +# Tests for src.logging.audit. # import logging @@ -16,7 +16,7 @@ import pytest -import api.logging.audit as audit +import src.logging.audit as audit # Do not run these tests alongside the rest of the test suite since # this tests adds an audit hook that interfere with other tests, diff --git a/app/tests/api/logging/test_flask_logger.py b/app/tests/src/logging/test_flask_logger.py similarity index 95% rename from app/tests/api/logging/test_flask_logger.py rename to app/tests/src/logging/test_flask_logger.py index c26250b8..7a6bf85c 100644 --- a/app/tests/api/logging/test_flask_logger.py +++ b/app/tests/src/logging/test_flask_logger.py @@ -4,13 +4,13 @@ import pytest from flask import Flask -import api.logging.flask_logger as flask_logger +import src.logging.flask_logger as flask_logger from tests.lib.assertions import assert_dict_contains @pytest.fixture def logger(): - logger = logging.getLogger("api") + logger = logging.getLogger("src") logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) return logger @@ -22,7 +22,7 @@ def app(logger): @app.get("/hello/") def hello(name): - logging.getLogger("api.hello").info(f"hello, {name}!") + logging.getLogger("src.hello").info(f"hello, {name}!") return "ok" flask_logger.init_app(logger, app) diff --git a/app/tests/api/logging/test_formatters.py b/app/tests/src/logging/test_formatters.py similarity index 97% rename from app/tests/api/logging/test_formatters.py rename to app/tests/src/logging/test_formatters.py index ecebe241..28b84d57 100644 --- a/app/tests/api/logging/test_formatters.py +++ b/app/tests/src/logging/test_formatters.py @@ -4,7 +4,7 @@ import pytest -import api.logging.formatters as formatters +import src.logging.formatters as formatters from tests.lib.assertions import assert_dict_contains diff --git a/app/tests/api/logging/test_logging.py b/app/tests/src/logging/test_logging.py similarity index 95% rename from app/tests/api/logging/test_logging.py rename to app/tests/src/logging/test_logging.py index 53d5a078..cb5ceba3 100644 --- a/app/tests/api/logging/test_logging.py +++ b/app/tests/src/logging/test_logging.py @@ -3,8 +3,8 @@ import pytest -import api.logging -import api.logging.formatters as formatters +import src.logging +import src.logging.formatters as formatters def _init_test_logger( @@ -12,7 +12,7 @@ def _init_test_logger( ): caplog.set_level(logging.DEBUG) monkeypatch.setenv("LOG_FORMAT", log_format) - api.logging.init("test_logging") + src.logging.init("test_logging") @pytest.fixture diff --git a/app/tests/api/logging/test_pii.py b/app/tests/src/logging/test_pii.py similarity index 96% rename from app/tests/api/logging/test_pii.py rename to app/tests/src/logging/test_pii.py index 67c4eedf..8b33652d 100644 --- a/app/tests/api/logging/test_pii.py +++ b/app/tests/src/logging/test_pii.py @@ -1,6 +1,6 @@ import pytest -import api.logging.pii as pii +import src.logging.pii as pii @pytest.mark.parametrize( diff --git a/app/tests/api/route/__init__.py b/app/tests/src/route/__init__.py similarity index 100% rename from app/tests/api/route/__init__.py rename to app/tests/src/route/__init__.py diff --git a/app/tests/api/route/test_healthcheck.py b/app/tests/src/route/test_healthcheck.py similarity index 94% rename from app/tests/api/route/test_healthcheck.py rename to app/tests/src/route/test_healthcheck.py index 32524e18..1fe7fd53 100644 --- a/app/tests/api/route/test_healthcheck.py +++ b/app/tests/src/route/test_healthcheck.py @@ -1,4 +1,4 @@ -import api.adapters.db as db +import src.adapters.db as db def test_get_healthcheck_200(client): diff --git a/app/tests/api/route/test_user_route.py b/app/tests/src/route/test_user_route.py similarity index 99% rename from app/tests/api/route/test_user_route.py rename to app/tests/src/route/test_user_route.py index 00a8b80a..f8b38efd 100644 --- a/app/tests/api/route/test_user_route.py +++ b/app/tests/src/route/test_user_route.py @@ -3,7 +3,7 @@ import faker import pytest -from tests.api.util.parametrize_utils import powerset +from tests.src.util.parametrize_utils import powerset fake = faker.Faker() diff --git a/app/tests/api/scripts/__init__.py b/app/tests/src/scripts/__init__.py similarity index 100% rename from app/tests/api/scripts/__init__.py rename to app/tests/src/scripts/__init__.py diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/src/scripts/test_create_user_csv.py similarity index 92% rename from app/tests/api/scripts/test_create_user_csv.py rename to app/tests/src/scripts/test_create_user_csv.py index ea615daa..a938eb79 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/src/scripts/test_create_user_csv.py @@ -7,13 +7,13 @@ from pytest_lazyfixture import lazy_fixture from smart_open import open as smart_open -import api.adapters.db as db -import api.app as app_entry -import tests.api.db.models.factories as factories -from api.db import models -from api.db.models.user_models import User -from tests.api.db.models.factories import UserFactory +import src.adapters.db as db +import src.app as app_entry +import tests.src.db.models.factories as factories +from src.db import models +from src.db.models.user_models import User from tests.lib import db_testing +from tests.src.db.models.factories import UserFactory @pytest.fixture diff --git a/app/tests/api/scripts/test_create_user_csv_expected.csv b/app/tests/src/scripts/test_create_user_csv_expected.csv similarity index 100% rename from app/tests/api/scripts/test_create_user_csv_expected.csv rename to app/tests/src/scripts/test_create_user_csv_expected.csv diff --git a/app/tests/api/util/__init__.py b/app/tests/src/util/__init__.py similarity index 100% rename from app/tests/api/util/__init__.py rename to app/tests/src/util/__init__.py diff --git a/app/tests/api/util/collections/__init__.py b/app/tests/src/util/collections/__init__.py similarity index 100% rename from app/tests/api/util/collections/__init__.py rename to app/tests/src/util/collections/__init__.py diff --git a/app/tests/api/util/collections/test_dict.py b/app/tests/src/util/collections/test_dict.py similarity index 92% rename from app/tests/api/util/collections/test_dict.py rename to app/tests/src/util/collections/test_dict.py index 1a8e98bb..242b4dc6 100644 --- a/app/tests/api/util/collections/test_dict.py +++ b/app/tests/src/util/collections/test_dict.py @@ -1,8 +1,8 @@ # -# Unit tests for api.util.collections.dict. +# Unit tests for src.util.collections.dict. # -import api.util.collections.dict as dict_util +import src.util.collections.dict as dict_util def test_least_recently_used_dict(): diff --git a/app/tests/api/util/parametrize_utils.py b/app/tests/src/util/parametrize_utils.py similarity index 100% rename from app/tests/api/util/parametrize_utils.py rename to app/tests/src/util/parametrize_utils.py diff --git a/app/tests/api/util/test_datetime_util.py b/app/tests/src/util/test_datetime_util.py similarity index 97% rename from app/tests/api/util/test_datetime_util.py rename to app/tests/src/util/test_datetime_util.py index 9a86200c..d6bf02d9 100644 --- a/app/tests/api/util/test_datetime_util.py +++ b/app/tests/src/util/test_datetime_util.py @@ -3,7 +3,7 @@ import pytest import pytz -from api.util.datetime_util import adjust_timezone +from src.util.datetime_util import adjust_timezone @pytest.mark.parametrize( diff --git a/docs/app/README.md b/docs/app/README.md index ba0f0742..43bcf83d 100644 --- a/docs/app/README.md +++ b/docs/app/README.md @@ -12,7 +12,7 @@ This is the API layer. It includes a few separate components: ```text root ├── app -│ └── api +│ └── src │ └── auth Authentication code for API │ └── db │ └── models DB model definitions @@ -68,7 +68,7 @@ Running in the native/local approach may require additional packages to be insta Most configuration options are managed by environment variables. -Environment variables for local development are stored in the [local.env](/app/local.env) file. This file is automatically loaded when running. If running within Docker, this file is specified as an `env_file` in the [docker-compose](/docker-compose.yml) file, and loaded [by a script](/app/api/util/local.py) automatically when running most other components outside the container. +Environment variables for local development are stored in the [local.env](/app/local.env) file. This file is automatically loaded when running. If running within Docker, this file is specified as an `env_file` in the [docker-compose](/docker-compose.yml) file, and loaded [by a script](/app/src/util/local.py) automatically when running most other components outside the container. Any environment variables specified directly in the [docker-compose](/docker-compose.yml) file will take precedent over those specified in the [local.env](/app/local.env) file. diff --git a/docs/app/api-details.md b/docs/app/api-details.md index 893199c8..9606e2fa 100644 --- a/docs/app/api-details.md +++ b/docs/app/api-details.md @@ -4,7 +4,7 @@ See [Technical Overview](./technical-overview.md) for details on the technologie Each endpoint is configured in the [openapi.yml](/app/openapi.yml) file which provides basic request validation. Each endpoint specifies an `operationId` that maps to a function defined in the code that will handle the request. -To make handling a request easier, an [ApiContext](/app/api/util/api_context.py) exists which will fetch the DB session, request body, and current user. This can be used like so: +To make handling a request easier, an [ApiContext](/app/src/util/api_context.py) exists which will fetch the DB session, request body, and current user. This can be used like so: ```py def example_post() -> flask.Response: with api_context_manager() as api_context: @@ -39,7 +39,7 @@ All model schemas defined can be found at the bottom of the UI. # Routes ## Health Check -[GET /v1/healthcheck](/app/api/route/healthcheck.py) is an endpoint for checking the health of the service. It verifies that the database is reachable, and that the API service itself is up and running. +[GET /v1/healthcheck](/app/src/route/healthcheck.py) is an endpoint for checking the health of the service. It verifies that the database is reachable, and that the API service itself is up and running. Note this endpoint explicitly does not require authorization so it can be integrated with any automated monitoring you may build. diff --git a/docs/app/database/database-access-management.md b/docs/app/database/database-access-management.md index 267a3987..60faa27f 100644 --- a/docs/app/database/database-access-management.md +++ b/docs/app/database/database-access-management.md @@ -4,7 +4,7 @@ This document describes the best practices and patterns for how the application ## Client Initialization and Configuration -The database client is initialized when the application starts (see [api/\_\_main\_\_.py](../../../app/api/__main__.py). As the database engine that is used to create acquire connections to the database is initialized using the database configuration defined in [db_config.py](../../../app/api/db/db_config.py), which is configured through environment variables. The initialized database client is then stored on the Flask app's [\`extensions\` dictionary](https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.extensions) to be used throughout the lifetime of the application. +The database client is initialized when the application starts (see [src/\_\_main\_\_.py](../../../app/src/__main__.py). As the database engine that is used to create acquire connections to the database is initialized using the database configuration defined in [db_config.py](../../../app/src/db/db_config.py), which is configured through environment variables. The initialized database client is then stored on the Flask app's [\`extensions\` dictionary](https://flask.palletsprojects.com/en/2.2.x/src/#flask.Flask.extensions) to be used throughout the lifetime of the application. ## Session Management @@ -16,7 +16,7 @@ For example, **do this** ### right way ### from flask import current_app -import api.adapters.db as db +import src.adapters.db as db def some_service_func(session: db.Session) with db_session.begin(): # start transaction @@ -35,7 +35,7 @@ and **don't do this** ### wrong way ### from flask import current_app -import api.adapters.db as db +import src.adapters.db as db def some_service_func() db_client = db.get_db(current_app) diff --git a/docs/app/database/database-migrations.md b/docs/app/database/database-migrations.md index ec16929b..31f89804 100644 --- a/docs/app/database/database-migrations.md +++ b/docs/app/database/database-migrations.md @@ -31,7 +31,7 @@ $ make db-upgrade
Example: Adding a new column to an existing table: -1. Manually update the database models with the changes ([example_models.py](/app/api/db/models/example_models.py) in this example) +1. Manually update the database models with the changes ([example_models.py](/app/src/db/models/example_models.py) in this example) ```python class ExampleTable(Base): ... diff --git a/docs/app/database/database-testing.md b/docs/app/database/database-testing.md index 1ccf9ff4..4b52b923 100644 --- a/docs/app/database/database-testing.md +++ b/docs/app/database/database-testing.md @@ -4,7 +4,7 @@ This document describes how the database is managed in the test suite. ## Test Schema -The test suite creates a new PostgreSQL database schema separate from the `public` schema that is used by the application outside of testing. This schema persists throughout the testing session is dropped at the end of the test run. The schema is created by the `db` fixture in [conftest.py](../../../app/tests/conftest.py). The fixture also creates and returns an initialized instance of the [db.DBClient](../../../app/api/db/__init__.py) that can be used to connect to the created schema. +The test suite creates a new PostgreSQL database schema separate from the `public` schema that is used by the application outside of testing. This schema persists throughout the testing session is dropped at the end of the test run. The schema is created by the `db` fixture in [conftest.py](../../../app/tests/conftest.py). The fixture also creates and returns an initialized instance of the [db.DBClient](../../../app/src/db/__init__.py) that can be used to connect to the created schema. Note that [PostgreSQL schemas](https://www.postgresql.org/docs/current/ddl-schemas.html) are entirely different concepts from [Schema objects in OpenAPI specification](https://swagger.io/docs/specification/data-models/). diff --git a/docs/app/formatting-and-linting.md b/docs/app/formatting-and-linting.md index b93b7919..bce9f557 100644 --- a/docs/app/formatting-and-linting.md +++ b/docs/app/formatting-and-linting.md @@ -4,7 +4,7 @@ Run `make format` to run all of the formatters. -When we run migrations via alembic, we autorun the formatters on the generated files. See [alembic.ini](/app/api/db/migrations/alembic.ini) for configuration. +When we run migrations via alembic, we autorun the formatters on the generated files. See [alembic.ini](/app/src/db/migrations/alembic.ini) for configuration. ### Isort [isort](https://pycqa.github.io/isort/) is used to sort our Python imports. Configuration options can be found in [pyproject.toml - tool.isort](/app/pyproject.toml) diff --git a/docs/app/monitoring-and-observability/logging-configuration.md b/docs/app/monitoring-and-observability/logging-configuration.md index 1a62e37b..993b048d 100644 --- a/docs/app/monitoring-and-observability/logging-configuration.md +++ b/docs/app/monitoring-and-observability/logging-configuration.md @@ -2,7 +2,7 @@ ## Overview -This document describes how logging is configured in the application. The logging functionality is defined in the [api.logging](../../../app/api/logging/) package and leverages Python's built-in [logging](https://docs.python.org/3/library/logging.html) framework. +This document describes how logging is configured in the application. The logging functionality is defined in the [src.logging](../../../app/src/logging/) package and leverages Python's built-in [logging](https://docs.python.org/3/library/logging.html) framework. ## Formatting @@ -12,7 +12,7 @@ We have two separate ways of formatting the logs which are controlled by the `LO ```json { - "name": "api.api.healthcheck", + "name": "src.api.healthcheck", "levelname": "INFO", "funcName": "healthcheck_get", "created": "1663261542.0465896", @@ -33,15 +33,15 @@ We have two separate ways of formatting the logs which are controlled by the `LO ## Logging Extra Data in a Request -The [api.logging.flask_logger](../../../app/api/logging/flask_logger.py) module adds logging functionality to Flask applications. It automatically adds useful data from the Flask request object to logs, logs the start and end of requests, and provides a mechanism for developers to dynamically add extra data to all subsequent logs for the current request. +The [src.logging.flask_logger](../../../app/src/logging/flask_logger.py) module adds logging functionality to Flask applications. It automatically adds useful data from the Flask request object to logs, logs the start and end of requests, and provides a mechanism for developers to dynamically add extra data to all subsequent logs for the current request. ## PII Masking -The api.logging.pii](../../../app/api/logging/pii.py) module defines a filter that applies to all logs that automatically masks data fields that look like social security numbers. +The src.logging.pii](../../../app/src/logging/pii.py) module defines a filter that applies to all logs that automatically masks data fields that look like social security numbers. ## Audit Logging -* The [api.logging.audit](../../../app/api/logging/audit.py) module defines a low level audit hook that logs events that may be of interest from a security point of view, such as dynamic code execution and network requests. +* The [src.logging.audit](../../../app/src/logging/audit.py) module defines a low level audit hook that logs events that may be of interest from a security point of view, such as dynamic code execution and network requests. ## Additional Reading diff --git a/docs/app/writing-tests.md b/docs/app/writing-tests.md index e94626f6..69f31bc9 100644 --- a/docs/app/writing-tests.md +++ b/docs/app/writing-tests.md @@ -16,7 +16,7 @@ For this project specifically: - All tests live under `app/tests/` - Under `tests/`, the organization mirrors the source code structure - - The tests for `app/api/route/` are found at `app/test/api/route/` + - The tests for `app/src/route/` are found at `app/test/api/route/` - Create `__init__.py` files for each directory. This helps [avoid name conflicts when pytest is resolving tests](https://docs.pytest.org/en/stable/goodpractices.html#tests-outside-application-code). @@ -66,7 +66,7 @@ They can be imported into tests from the path `tests.helpers`, for example, To facilitate easier setup of test data, most database models have factories via [factory_boy](https://factoryboy.readthedocs.io/) in -`app/api/db/models/factories.py`. +`app/src/db/models/factories.py`. There are a few different ways of [using the factories](https://factoryboy.readthedocs.io/en/stable/#using-factories), termed @@ -92,4 +92,4 @@ FooFactory.build(foo_id=5, name="Bar") ``` would set `foo_id=5` and `name="Bar"` on the generated model, while all other -attributes would use what's configured on the factory class. \ No newline at end of file +attributes would use what's configured on the factory class. From 45b9e3f696bbdf95191e4ef2095f6afcfbab49eb Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Thu, 16 Feb 2023 15:34:03 -0800 Subject: [PATCH 34/51] Remove rollback logic from test --- app/tests/conftest.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 66a8e1e7..3fac7b0b 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -79,32 +79,16 @@ def empty_schema(monkeypatch) -> db.DBClient: @pytest.fixture -def test_db_session(db_client: db.DBClient) -> db.Session: - # Based on https://docs.sqlalchemy.org/en/13/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites - with db_client.get_connection() as connection: - trans = connection.begin() - - # Rather than call db.get_session() to create a new session with a new connection, - # create a session bound to the existing connection that has a transaction manually start. - # This allows the transaction to be rolled back after the test completes. - with db.Session(bind=connection, autocommit=False, expire_on_commit=False) as session: - session.begin_nested() - - @sqlalchemy.event.listens_for(session, "after_transaction_end") - def restart_savepoint(session, transaction): - if transaction.nested and not transaction._parent.nested: - session.begin_nested() - - yield session - - trans.rollback() +def db_session(db_client: db.DBClient) -> db.Session: + with db_client.get_session() as session: + yield session @pytest.fixture -def factories_db_session(monkeypatch, test_db_session) -> db.Session: - monkeypatch.setattr(factories, "_db_session", test_db_session) - logger.info("set factories db_session to %s", test_db_session) - return test_db_session +def factories_db_session(monkeypatch, db_session) -> db.Session: + monkeypatch.setattr(factories, "_db_session", db_session) + logger.info("set factories db_session to %s", db_session) + return db_session #################### From e55bd99379b2c7fd642a8ac35df8a811c21ce089 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 09:31:27 -0800 Subject: [PATCH 35/51] Revert "Remove rollback logic from test" This reverts commit 45b9e3f696bbdf95191e4ef2095f6afcfbab49eb. --- app/tests/conftest.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 3fac7b0b..66a8e1e7 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -79,16 +79,32 @@ def empty_schema(monkeypatch) -> db.DBClient: @pytest.fixture -def db_session(db_client: db.DBClient) -> db.Session: - with db_client.get_session() as session: - yield session +def test_db_session(db_client: db.DBClient) -> db.Session: + # Based on https://docs.sqlalchemy.org/en/13/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites + with db_client.get_connection() as connection: + trans = connection.begin() + + # Rather than call db.get_session() to create a new session with a new connection, + # create a session bound to the existing connection that has a transaction manually start. + # This allows the transaction to be rolled back after the test completes. + with db.Session(bind=connection, autocommit=False, expire_on_commit=False) as session: + session.begin_nested() + + @sqlalchemy.event.listens_for(session, "after_transaction_end") + def restart_savepoint(session, transaction): + if transaction.nested and not transaction._parent.nested: + session.begin_nested() + + yield session + + trans.rollback() @pytest.fixture -def factories_db_session(monkeypatch, db_session) -> db.Session: - monkeypatch.setattr(factories, "_db_session", db_session) - logger.info("set factories db_session to %s", db_session) - return db_session +def factories_db_session(monkeypatch, test_db_session) -> db.Session: + monkeypatch.setattr(factories, "_db_session", test_db_session) + logger.info("set factories db_session to %s", test_db_session) + return test_db_session #################### From 5622433336b8a54332fe48dd15a4c1c8f1fe4613 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 09:31:31 -0800 Subject: [PATCH 36/51] Revert "Rename source directory from api to src" This reverts commit 0dc857f8d4b4ad36a7e4b05c0bc398b248e42b32. --- app/Dockerfile | 2 +- app/Makefile | 22 +++++++++---------- app/{src => api}/__init__.py | 0 app/{src => api}/__main__.py | 10 ++++----- app/{src => api}/adapters/db/__init__.py | 4 ++-- app/{src => api}/adapters/db/client.py | 2 +- app/{src => api}/adapters/db/config.py | 2 +- app/{src => api}/adapters/db/flask_db.py | 16 +++++++------- app/{src => api}/api/__init__.py | 0 app/{src => api}/api/healthcheck.py | 6 ++--- app/{src => api}/api/response.py | 4 ++-- app/{src => api}/api/schemas/__init__.py | 0 .../api/schemas/request_schema.py | 0 .../api/schemas/response_schema.py | 2 +- app/{src => api}/api/users/__init__.py | 6 ++--- app/{src => api}/api/users/user_blueprint.py | 0 app/{src => api}/api/users/user_commands.py | 10 ++++----- app/{src => api}/api/users/user_routes.py | 18 +++++++-------- app/{src => api}/api/users/user_schemas.py | 4 ++-- app/{src => api}/app.py | 18 +++++++-------- app/{src => api}/app_config.py | 2 +- app/{src => api}/auth/__init__.py | 0 app/{src => api}/auth/api_key_auth.py | 0 app/{src => api}/db/__init__.py | 0 app/{src => api}/db/migrations/__init__.py | 0 app/{src => api}/db/migrations/alembic.ini | 2 +- app/{src => api}/db/migrations/env.py | 14 ++++++------ app/{src => api}/db/migrations/run.py | 0 app/{src => api}/db/migrations/script.py.mako | 0 .../2022_12_16_create_user_and_role_tables.py | 0 app/{src => api}/db/models/__init__.py | 0 app/{src => api}/db/models/base.py | 2 +- app/{src => api}/db/models/user_models.py | 2 +- app/{src => api}/logging/__init__.py | 10 ++++----- app/{src => api}/logging/audit.py | 4 ++-- app/{src => api}/logging/config.py | 10 ++++----- app/{src => api}/logging/decodelog.py | 2 +- app/{src => api}/logging/flask_logger.py | 8 +++---- app/{src => api}/logging/formatters.py | 2 +- app/{src => api}/logging/pii.py | 4 ++-- app/{src => api}/services/users/__init__.py | 0 .../services/users/create_user.py | 6 ++--- .../services/users/create_user_csv.py | 4 ++-- app/{src => api}/services/users/get_user.py | 4 ++-- app/{src => api}/services/users/patch_user.py | 6 ++--- app/{src => api}/util/__init__.py | 0 app/{src => api}/util/collections/__init__.py | 0 app/{src => api}/util/collections/dict.py | 0 app/{src => api}/util/datetime_util.py | 0 app/{src => api}/util/env_config.py | 4 ++-- app/{src => api}/util/file_util.py | 0 app/{src => api}/util/local.py | 0 app/{src => api}/util/string_utils.py | 0 app/local.env | 4 ++-- app/pyproject.toml | 14 ++++++------ app/tests/{src => api}/__init__.py | 0 app/tests/{src => api}/adapters/__init__.py | 0 app/tests/{src => api}/adapters/db/test_db.py | 6 ++--- .../{src => api}/adapters/db/test_flask_db.py | 4 ++-- .../{src => api}/auth/test_api_key_auth.py | 2 +- app/tests/{src => api}/db/__init__.py | 0 app/tests/{src => api}/db/models/__init__.py | 0 app/tests/{src => api}/db/models/factories.py | 8 +++---- .../{src => api}/db/models/test_factories.py | 4 ++-- app/tests/{src => api}/db/test_migrations.py | 2 +- app/tests/{src => api}/logging/__init__.py | 0 app/tests/{src => api}/logging/test_audit.py | 4 ++-- .../{src => api}/logging/test_flask_logger.py | 6 ++--- .../{src => api}/logging/test_formatters.py | 2 +- .../{src => api}/logging/test_logging.py | 6 ++--- app/tests/{src => api}/logging/test_pii.py | 2 +- app/tests/{src => api}/route/__init__.py | 0 .../{src => api}/route/test_healthcheck.py | 2 +- .../{src => api}/route/test_user_route.py | 2 +- app/tests/{src => api}/scripts/__init__.py | 0 .../scripts/test_create_user_csv.py | 12 +++++----- .../scripts/test_create_user_csv_expected.csv | 0 app/tests/{src => api}/util/__init__.py | 0 .../{src => api}/util/collections/__init__.py | 0 .../util/collections/test_dict.py | 4 ++-- .../{src => api}/util/parametrize_utils.py | 0 .../{src => api}/util/test_datetime_util.py | 2 +- app/tests/conftest.py | 10 ++++----- app/tests/lib/db_testing.py | 4 ++-- docs/app/README.md | 4 ++-- docs/app/api-details.md | 4 ++-- .../database/database-access-management.md | 6 ++--- docs/app/database/database-migrations.md | 2 +- docs/app/database/database-testing.md | 2 +- docs/app/formatting-and-linting.md | 2 +- .../logging-configuration.md | 10 ++++----- docs/app/writing-tests.md | 6 ++--- 92 files changed, 174 insertions(+), 174 deletions(-) rename app/{src => api}/__init__.py (100%) rename app/{src => api}/__main__.py (88%) rename app/{src => api}/adapters/db/__init__.py (89%) rename app/{src => api}/adapters/db/client.py (99%) rename app/{src => api}/adapters/db/config.py (95%) rename app/{src => api}/adapters/db/flask_db.py (88%) rename app/{src => api}/api/__init__.py (100%) rename app/{src => api}/api/healthcheck.py (89%) rename app/{src => api}/api/response.py (95%) rename app/{src => api}/api/schemas/__init__.py (100%) rename app/{src => api}/api/schemas/request_schema.py (100%) rename app/{src => api}/api/schemas/response_schema.py (95%) rename app/{src => api}/api/users/__init__.py (51%) rename app/{src => api}/api/users/user_blueprint.py (100%) rename app/{src => api}/api/users/user_commands.py (80%) rename app/{src => api}/api/users/user_routes.py (83%) rename app/{src => api}/api/users/user_schemas.py (94%) rename app/{src => api}/app.py (83%) rename app/{src => api}/app_config.py (89%) rename app/{src => api}/auth/__init__.py (100%) rename app/{src => api}/auth/api_key_auth.py (100%) rename app/{src => api}/db/__init__.py (100%) rename app/{src => api}/db/migrations/__init__.py (100%) rename app/{src => api}/db/migrations/alembic.ini (97%) rename app/{src => api}/db/migrations/env.py (89%) rename app/{src => api}/db/migrations/run.py (100%) rename app/{src => api}/db/migrations/script.py.mako (100%) rename app/{src => api}/db/migrations/versions/2022_12_16_create_user_and_role_tables.py (100%) rename app/{src => api}/db/models/__init__.py (100%) rename app/{src => api}/db/models/base.py (98%) rename app/{src => api}/db/models/user_models.py (96%) rename app/{src => api}/logging/__init__.py (91%) rename app/{src => api}/logging/audit.py (97%) rename app/{src => api}/logging/config.py (93%) rename app/{src => api}/logging/decodelog.py (99%) rename app/{src => api}/logging/flask_logger.py (95%) rename app/{src => api}/logging/formatters.py (97%) rename app/{src => api}/logging/pii.py (97%) rename app/{src => api}/services/users/__init__.py (100%) rename app/{src => api}/services/users/create_user.py (90%) rename app/{src => api}/services/users/create_user_csv.py (96%) rename app/{src => api}/services/users/get_user.py (91%) rename app/{src => api}/services/users/patch_user.py (94%) rename app/{src => api}/util/__init__.py (100%) rename app/{src => api}/util/collections/__init__.py (100%) rename app/{src => api}/util/collections/dict.py (100%) rename app/{src => api}/util/datetime_util.py (100%) rename app/{src => api}/util/env_config.py (78%) rename app/{src => api}/util/file_util.py (100%) rename app/{src => api}/util/local.py (100%) rename app/{src => api}/util/string_utils.py (100%) rename app/tests/{src => api}/__init__.py (100%) rename app/tests/{src => api}/adapters/__init__.py (100%) rename app/tests/{src => api}/adapters/db/test_db.py (96%) rename app/tests/{src => api}/adapters/db/test_flask_db.py (93%) rename app/tests/{src => api}/auth/test_api_key_auth.py (93%) rename app/tests/{src => api}/db/__init__.py (100%) rename app/tests/{src => api}/db/models/__init__.py (100%) rename app/tests/{src => api}/db/models/factories.py (93%) rename app/tests/{src => api}/db/models/test_factories.py (96%) rename app/tests/{src => api}/db/test_migrations.py (96%) rename app/tests/{src => api}/logging/__init__.py (100%) rename app/tests/{src => api}/logging/test_audit.py (99%) rename app/tests/{src => api}/logging/test_flask_logger.py (95%) rename app/tests/{src => api}/logging/test_formatters.py (97%) rename app/tests/{src => api}/logging/test_logging.py (95%) rename app/tests/{src => api}/logging/test_pii.py (96%) rename app/tests/{src => api}/route/__init__.py (100%) rename app/tests/{src => api}/route/test_healthcheck.py (94%) rename app/tests/{src => api}/route/test_user_route.py (99%) rename app/tests/{src => api}/scripts/__init__.py (100%) rename app/tests/{src => api}/scripts/test_create_user_csv.py (92%) rename app/tests/{src => api}/scripts/test_create_user_csv_expected.csv (100%) rename app/tests/{src => api}/util/__init__.py (100%) rename app/tests/{src => api}/util/collections/__init__.py (100%) rename app/tests/{src => api}/util/collections/test_dict.py (92%) rename app/tests/{src => api}/util/parametrize_utils.py (100%) rename app/tests/{src => api}/util/test_datetime_util.py (97%) diff --git a/app/Dockerfile b/app/Dockerfile index 9ec44fbb..6b944494 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -34,4 +34,4 @@ ENV HOST=0.0.0.0 # https://python-poetry.org/docs/basic-usage/#installing-dependencies RUN poetry install # Run the application. -CMD ["poetry", "run", "python", "-m", "src"] +CMD ["poetry", "run", "python", "-m", "api"] diff --git a/app/Makefile b/app/Makefile index 244c3c99..41fea3cd 100644 --- a/app/Makefile +++ b/app/Makefile @@ -10,15 +10,15 @@ APP_NAME := main-app # Note that you can also change the LOG_FORMAT env var to switch # between JSON & human readable format. This is left in place # in the event JSON is output from a process we don't log. -DECODE_LOG := 2>&1 | python3 -u src/logging/util/decodelog.py +DECODE_LOG := 2>&1 | python3 -u api/logging/util/decodelog.py # A few commands need adjustments if they're run in CI, specify those here # TODO - when CI gets hooked up, actually test this. ifdef CI DOCKER_EXEC_ARGS := -T -e CI -e PYTEST_ADDOPTS="--color=yes" - FLAKE8_FORMAT := '::warning file=src/%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s' + FLAKE8_FORMAT := '::warning file=api/%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s' MYPY_FLAGS := --no-pretty - MYPY_POSTPROC := | perl -pe "s/^(.+):(\d+):(\d+): error: (.*)/::warning file=src\/\1,line=\2,col=\3::\4/" + MYPY_POSTPROC := | perl -pe "s/^(.+):(\d+):(\d+): error: (.*)/::warning file=api\/\1,line=\2,col=\3::\4/" else FLAKE8_FORMAT := default endif @@ -92,7 +92,7 @@ db-recreate: clean-docker-volumes init-db # DB Migrations ######################### -alembic_config := ./src/db/migrations/alembic.ini +alembic_config := ./api/db/migrations/alembic.ini alembic_cmd := $(PY_RUN_CMD) alembic --config $(alembic_config) db-upgrade: ## Apply pending migrations to db @@ -139,7 +139,7 @@ test-watch: $(PY_RUN_CMD) pytest-watch --clear $(args) test-coverage: - $(PY_RUN_CMD) coverage run --branch --source=src -m pytest -m "not audit" $(args) + $(PY_RUN_CMD) coverage run --branch --source=api -m pytest -m "not audit" $(args) $(PY_RUN_CMD) coverage report test-coverage-report: ## Open HTML test coverage report @@ -151,22 +151,22 @@ test-coverage-report: ## Open HTML test coverage report ################################################## format: - $(PY_RUN_CMD) isort --atomic src tests - $(PY_RUN_CMD) black src tests + $(PY_RUN_CMD) isort --atomic api tests + $(PY_RUN_CMD) black api tests format-check: - $(PY_RUN_CMD) isort --atomic --check-only src tests - $(PY_RUN_CMD) black --check src tests + $(PY_RUN_CMD) isort --atomic --check-only api tests + $(PY_RUN_CMD) black --check api tests lint: lint-py lint-py: lint-flake lint-mypy lint-poetry-version lint-flake: - $(PY_RUN_CMD) flake8 --format=$(FLAKE8_FORMAT) src tests + $(PY_RUN_CMD) flake8 --format=$(FLAKE8_FORMAT) api tests lint-mypy: - $(PY_RUN_CMD) mypy --show-error-codes $(MYPY_FLAGS) src $(MYPY_POSTPROC) + $(PY_RUN_CMD) mypy --show-error-codes $(MYPY_FLAGS) api $(MYPY_POSTPROC) lint-poetry-version: ## Check poetry version grep --quiet 'lock-version = "1.1"' poetry.lock diff --git a/app/src/__init__.py b/app/api/__init__.py similarity index 100% rename from app/src/__init__.py rename to app/api/__init__.py diff --git a/app/src/__main__.py b/app/api/__main__.py similarity index 88% rename from app/src/__main__.py rename to app/api/__main__.py index 725a6f2d..aab01a51 100644 --- a/app/src/__main__.py +++ b/app/api/__main__.py @@ -7,10 +7,10 @@ import logging -import src.app -import src.logging -from src.app_config import AppConfig -from src.util.local import load_local_env_vars +import api.app +import api.logging +from api.app_config import AppConfig +from api.util.local import load_local_env_vars logger = logging.getLogger(__package__) @@ -19,7 +19,7 @@ def main() -> None: load_local_env_vars() app_config = AppConfig() - app = src.app.create_app() + app = api.app.create_app() environment = app_config.environment diff --git a/app/src/adapters/db/__init__.py b/app/api/adapters/db/__init__.py similarity index 89% rename from app/src/adapters/db/__init__.py rename to app/api/adapters/db/__init__.py index ddb68e4e..9f415607 100644 --- a/app/src/adapters/db/__init__.py +++ b/app/api/adapters/db/__init__.py @@ -7,7 +7,7 @@ To use this module with Flask, use the flask_db module. Usage: - import src.adapters.db as db + import api.adapters.db as db db_client = db.init() @@ -23,7 +23,7 @@ """ # Re-export for convenience -from src.adapters.db.client import Connection, DBClient, Session, init +from api.adapters.db.client import Connection, DBClient, Session, init # Do not import flask_db here, because this module is not dependent on any specific framework. # Code can choose to use this module on its own or with the flask_db module depending on needs. diff --git a/app/src/adapters/db/client.py b/app/api/adapters/db/client.py similarity index 99% rename from app/src/adapters/db/client.py rename to app/api/adapters/db/client.py index 3f55e357..36f38eee 100644 --- a/app/src/adapters/db/client.py +++ b/app/api/adapters/db/client.py @@ -17,7 +17,7 @@ import sqlalchemy.pool as pool from sqlalchemy.orm import session -from src.adapters.db.config import DbConfig, get_db_config +from api.adapters.db.config import DbConfig, get_db_config # Re-export the Connection type that is returned by the get_connection() method # to be used for type hints. diff --git a/app/src/adapters/db/config.py b/app/api/adapters/db/config.py similarity index 95% rename from app/src/adapters/db/config.py rename to app/api/adapters/db/config.py index dff1d91f..67475f65 100644 --- a/app/src/adapters/db/config.py +++ b/app/api/adapters/db/config.py @@ -3,7 +3,7 @@ from pydantic import Field -from src.util.env_config import PydanticBaseEnvConfig +from api.util.env_config import PydanticBaseEnvConfig logger = logging.getLogger(__name__) diff --git a/app/src/adapters/db/flask_db.py b/app/api/adapters/db/flask_db.py similarity index 88% rename from app/src/adapters/db/flask_db.py rename to app/api/adapters/db/flask_db.py index 24960958..04f486c3 100644 --- a/app/src/adapters/db/flask_db.py +++ b/app/api/adapters/db/flask_db.py @@ -5,8 +5,8 @@ of a Flask app and an instance of a DBClient. Example: - import src.adapters.db as db - import src.adapters.db.flask_db as flask_db + import api.adapters.db as db + import api.adapters.db.flask_db as flask_db db_client = db.init() app = APIFlask(__name__) @@ -16,8 +16,8 @@ new database session that lasts for the duration of the request. Example: - import src.adapters.db as db - import src.adapters.db.flask_db as flask_db + import api.adapters.db as db + import api.adapters.db.flask_db as flask_db @app.route("/health") @flask_db.with_db_session @@ -31,7 +31,7 @@ def health(db_session: db.Session): Example: from flask import current_app - import src.adapters.db.flask_db as flask_db + import api.adapters.db.flask_db as flask_db @app.route("/health") def health(): @@ -43,8 +43,8 @@ def health(): from flask import Flask, current_app -import src.adapters.db as db -from src.adapters.db.client import DBClient +import api.adapters.db as db +from api.adapters.db.client import DBClient _FLASK_EXTENSION_KEY = "db" @@ -67,7 +67,7 @@ def get_db(app: Flask) -> DBClient: Example: from flask import current_app - import src.adapters.db.flask_db as flask_db + import api.adapters.db.flask_db as flask_db @app.route("/health") def health(): diff --git a/app/src/api/__init__.py b/app/api/api/__init__.py similarity index 100% rename from app/src/api/__init__.py rename to app/api/api/__init__.py diff --git a/app/src/api/healthcheck.py b/app/api/api/healthcheck.py similarity index 89% rename from app/src/api/healthcheck.py rename to app/api/api/healthcheck.py index 4dc5782b..4df442b2 100644 --- a/app/src/api/healthcheck.py +++ b/app/api/api/healthcheck.py @@ -6,9 +6,9 @@ from sqlalchemy import text from werkzeug.exceptions import ServiceUnavailable -import src.adapters.db.flask_db as flask_db -from src.api import response -from src.api.schemas import request_schema +import api.adapters.db.flask_db as flask_db +from api.api import response +from api.api.schemas import request_schema logger = logging.getLogger(__name__) diff --git a/app/src/api/response.py b/app/api/api/response.py similarity index 95% rename from app/src/api/response.py rename to app/api/api/response.py index 70e2ae8f..65f6e0bb 100644 --- a/app/src/api/response.py +++ b/app/api/api/response.py @@ -1,8 +1,8 @@ import dataclasses from typing import Optional -from src.api.schemas import response_schema -from src.db.models.base import Base +from api.api.schemas import response_schema +from api.db.models.base import Base @dataclasses.dataclass diff --git a/app/src/api/schemas/__init__.py b/app/api/api/schemas/__init__.py similarity index 100% rename from app/src/api/schemas/__init__.py rename to app/api/api/schemas/__init__.py diff --git a/app/src/api/schemas/request_schema.py b/app/api/api/schemas/request_schema.py similarity index 100% rename from app/src/api/schemas/request_schema.py rename to app/api/api/schemas/request_schema.py diff --git a/app/src/api/schemas/response_schema.py b/app/api/api/schemas/response_schema.py similarity index 95% rename from app/src/api/schemas/response_schema.py rename to app/api/api/schemas/response_schema.py index 5d89e85e..9ae6f6ac 100644 --- a/app/src/api/schemas/response_schema.py +++ b/app/api/api/schemas/response_schema.py @@ -1,6 +1,6 @@ from apiflask import fields -from src.api.schemas import request_schema +from api.api.schemas import request_schema class ValidationErrorSchema(request_schema.OrderedSchema): diff --git a/app/src/api/users/__init__.py b/app/api/api/users/__init__.py similarity index 51% rename from app/src/api/users/__init__.py rename to app/api/api/users/__init__.py index 8ddfd549..1da0ac9f 100644 --- a/app/src/api/users/__init__.py +++ b/app/api/api/users/__init__.py @@ -1,10 +1,10 @@ -from src.api.users.user_blueprint import user_blueprint +from api.api.users.user_blueprint import user_blueprint # import user_commands module to register the CLI commands on the user_blueprint -import src.api.users.user_commands # noqa: F401 E402 isort:skip +import api.api.users.user_commands # noqa: F401 E402 isort:skip # import user_commands module to register the API routes on the user_blueprint -import src.api.users.user_routes # noqa: F401 E402 isort:skip +import api.api.users.user_routes # noqa: F401 E402 isort:skip __all__ = ["user_blueprint"] diff --git a/app/src/api/users/user_blueprint.py b/app/api/api/users/user_blueprint.py similarity index 100% rename from app/src/api/users/user_blueprint.py rename to app/api/api/users/user_blueprint.py diff --git a/app/src/api/users/user_commands.py b/app/api/api/users/user_commands.py similarity index 80% rename from app/src/api/users/user_commands.py rename to app/api/api/users/user_commands.py index 71f23f57..515bd4b4 100644 --- a/app/src/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -4,11 +4,11 @@ import click -import src.adapters.db as db -import src.adapters.db.flask_db as flask_db -import src.services.users as user_service -from src.api.users.user_blueprint import user_blueprint -from src.util.datetime_util import utcnow +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db +import api.services.users as user_service +from api.api.users.user_blueprint import user_blueprint +from api.util.datetime_util import utcnow logger = logging.getLogger(__name__) diff --git a/app/src/api/users/user_routes.py b/app/api/api/users/user_routes.py similarity index 83% rename from app/src/api/users/user_routes.py rename to app/api/api/users/user_routes.py index 72163d72..5ef77eee 100644 --- a/app/src/api/users/user_routes.py +++ b/app/api/api/users/user_routes.py @@ -1,15 +1,15 @@ import logging from typing import Any -import src.adapters.db as db -import src.adapters.db.flask_db as flask_db -import src.api.response as response -import src.api.users.user_schemas as user_schemas -import src.services.users as user_service -import src.services.users as users -from src.api.users.user_blueprint import user_blueprint -from src.auth.api_key_auth import api_key_auth -from src.db.models.user_models import User +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db +import api.api.response as response +import api.api.users.user_schemas as user_schemas +import api.services.users as user_service +import api.services.users as users +from api.api.users.user_blueprint import user_blueprint +from api.auth.api_key_auth import api_key_auth +from api.db.models.user_models import User logger = logging.getLogger(__name__) diff --git a/app/src/api/users/user_schemas.py b/app/api/api/users/user_schemas.py similarity index 94% rename from app/src/api/users/user_schemas.py rename to app/api/api/users/user_schemas.py index b0fd2c14..cc44c6b1 100644 --- a/app/src/api/users/user_schemas.py +++ b/app/api/api/users/user_schemas.py @@ -1,8 +1,8 @@ from apiflask import fields from marshmallow import fields as marshmallow_fields -from src.api.schemas import request_schema -from src.db.models import user_models +from api.api.schemas import request_schema +from api.db.models import user_models class RoleSchema(request_schema.OrderedSchema): diff --git a/app/src/app.py b/app/api/app.py similarity index 83% rename from app/src/app.py rename to app/api/app.py index f1d72806..4c118c90 100644 --- a/app/src/app.py +++ b/app/api/app.py @@ -6,14 +6,14 @@ from flask import g from werkzeug.exceptions import Unauthorized -import src.adapters.db as db -import src.adapters.db.flask_db as flask_db -import src.logging -import src.logging.flask_logger as flask_logger -from src.api.healthcheck import healthcheck_blueprint -from src.api.schemas import response_schema -from src.api.users import user_blueprint -from src.auth.api_key_auth import User, get_app_security_scheme +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db +import api.logging +import api.logging.flask_logger as flask_logger +from api.api.healthcheck import healthcheck_blueprint +from api.api.schemas import response_schema +from api.api.users import user_blueprint +from api.auth.api_key_auth import User, get_app_security_scheme logger = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def create_app() -> APIFlask: app = APIFlask(__name__) - root_logger = src.logging.init(__package__) + root_logger = api.logging.init(__package__) flask_logger.init_app(root_logger, app) db_client = db.init() diff --git a/app/src/app_config.py b/app/api/app_config.py similarity index 89% rename from app/src/app_config.py rename to app/api/app_config.py index 44159955..e97be25f 100644 --- a/app/src/app_config.py +++ b/app/api/app_config.py @@ -1,4 +1,4 @@ -from src.util.env_config import PydanticBaseEnvConfig +from api.util.env_config import PydanticBaseEnvConfig class AppConfig(PydanticBaseEnvConfig): diff --git a/app/src/auth/__init__.py b/app/api/auth/__init__.py similarity index 100% rename from app/src/auth/__init__.py rename to app/api/auth/__init__.py diff --git a/app/src/auth/api_key_auth.py b/app/api/auth/api_key_auth.py similarity index 100% rename from app/src/auth/api_key_auth.py rename to app/api/auth/api_key_auth.py diff --git a/app/src/db/__init__.py b/app/api/db/__init__.py similarity index 100% rename from app/src/db/__init__.py rename to app/api/db/__init__.py diff --git a/app/src/db/migrations/__init__.py b/app/api/db/migrations/__init__.py similarity index 100% rename from app/src/db/migrations/__init__.py rename to app/api/db/migrations/__init__.py diff --git a/app/src/db/migrations/alembic.ini b/app/api/db/migrations/alembic.ini similarity index 97% rename from app/src/db/migrations/alembic.ini rename to app/api/db/migrations/alembic.ini index 7f64607f..8850aced 100644 --- a/app/src/db/migrations/alembic.ini +++ b/app/api/db/migrations/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = src/db/migrations +script_location = api/db/migrations file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(slug)s diff --git a/app/src/db/migrations/env.py b/app/api/db/migrations/env.py similarity index 89% rename from app/src/db/migrations/env.py rename to app/api/db/migrations/env.py index e519705a..50b105f1 100644 --- a/app/src/db/migrations/env.py +++ b/app/api/db/migrations/env.py @@ -6,19 +6,19 @@ from alembic import context # Alembic cli seems to reset the path on load causing issues with local module imports. -# Workaround is to force set the path to the current run directory (top level src folder) +# Workaround is to force set the path to the current run directory (top level api folder) # See database migrations section in `./database/database-migrations.md` for details about running migrations. sys.path.insert(0, ".") # noqa: E402 # Load env vars before anything further -from src.util.local import load_local_env_vars # noqa: E402 isort:skip +from api.util.local import load_local_env_vars # noqa: E402 isort:skip load_local_env_vars() -from src.adapters.db.client import make_connection_uri # noqa: E402 isort:skip -from src.adapters.db.config import get_db_config # noqa: E402 isort:skip -from src.db.models import metadata # noqa: E402 isort:skip -import src.logging # noqa: E402 isort:skip +from api.adapters.db.client import make_connection_uri # noqa: E402 isort:skip +from api.adapters.db.config import get_db_config # noqa: E402 isort:skip +from api.db.models import metadata # noqa: E402 isort:skip +import api.logging # noqa: E402 isort:skip # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -27,7 +27,7 @@ logger = logging.getLogger("migrations") # Initialize logging -src.logging.init("migrations") +api.logging.init("migrations") if not config.get_main_option("sqlalchemy.url"): uri = make_connection_uri(get_db_config()) diff --git a/app/src/db/migrations/run.py b/app/api/db/migrations/run.py similarity index 100% rename from app/src/db/migrations/run.py rename to app/api/db/migrations/run.py diff --git a/app/src/db/migrations/script.py.mako b/app/api/db/migrations/script.py.mako similarity index 100% rename from app/src/db/migrations/script.py.mako rename to app/api/db/migrations/script.py.mako diff --git a/app/src/db/migrations/versions/2022_12_16_create_user_and_role_tables.py b/app/api/db/migrations/versions/2022_12_16_create_user_and_role_tables.py similarity index 100% rename from app/src/db/migrations/versions/2022_12_16_create_user_and_role_tables.py rename to app/api/db/migrations/versions/2022_12_16_create_user_and_role_tables.py diff --git a/app/src/db/models/__init__.py b/app/api/db/models/__init__.py similarity index 100% rename from app/src/db/models/__init__.py rename to app/api/db/models/__init__.py diff --git a/app/src/db/models/base.py b/app/api/db/models/base.py similarity index 98% rename from app/src/db/models/base.py rename to app/api/db/models/base.py index 6cb4d9b8..111f688e 100644 --- a/app/src/db/models/base.py +++ b/app/api/db/models/base.py @@ -10,7 +10,7 @@ from sqlalchemy.orm import declarative_mixin from sqlalchemy.sql.functions import now as sqlnow -from src.util import datetime_util +from api.util import datetime_util # Override the default naming of constraints # to use suffixes instead: diff --git a/app/src/db/models/user_models.py b/app/api/db/models/user_models.py similarity index 96% rename from app/src/db/models/user_models.py rename to app/api/db/models/user_models.py index 6631f3ee..47341160 100644 --- a/app/src/db/models/user_models.py +++ b/app/api/db/models/user_models.py @@ -8,7 +8,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Mapped, relationship -from src.db.models.base import Base, IdMixin, TimestampMixin +from api.db.models.base import Base, IdMixin, TimestampMixin logger = logging.getLogger(__name__) diff --git a/app/src/logging/__init__.py b/app/api/logging/__init__.py similarity index 91% rename from app/src/logging/__init__.py rename to app/api/logging/__init__.py index 463b4eec..f1178bc6 100644 --- a/app/src/logging/__init__.py +++ b/app/api/logging/__init__.py @@ -3,15 +3,15 @@ There are two formatters for the log messages: human-readable and JSON. The formatter that is used is determined by the environment variable LOG_FORMAT. If the environment variable is not set, the JSON formatter -is used by default. See src.logging.formatters for more information. +is used by default. See api.logging.formatters for more information. The logger also adds a PII mask filter to the root logger. See -src.logging.pii for more information. +api.logging.pii for more information. Usage: - import src.logging + import api.logging - src.logging.init("program name") + api.logging.init("program name") Once the module has been initialized, the standard logging module can be used to log messages: @@ -30,7 +30,7 @@ import sys from typing import Any, cast -import src.logging.config as config +import api.logging.config as config logger = logging.getLogger(__name__) _original_argv = tuple(sys.argv) diff --git a/app/src/logging/audit.py b/app/api/logging/audit.py similarity index 97% rename from app/src/logging/audit.py rename to app/api/logging/audit.py index 91a8f33b..a203992b 100644 --- a/app/src/logging/audit.py +++ b/app/api/logging/audit.py @@ -10,7 +10,7 @@ import sys from typing import Any, Sequence -import src.util.collections +import api.util.collections logger = logging.getLogger(__name__) @@ -95,4 +95,4 @@ def log_audit_event(event_name: str, args: Sequence[Any], arg_names: Sequence[st logger.log(AUDIT, event_name, extra=extra) -audit_message_count = src.util.collections.LeastRecentlyUsedDict() +audit_message_count = api.util.collections.LeastRecentlyUsedDict() diff --git a/app/src/logging/config.py b/app/api/logging/config.py similarity index 93% rename from app/src/logging/config.py rename to app/api/logging/config.py index 46ac7324..d58e02af 100644 --- a/app/src/logging/config.py +++ b/app/api/logging/config.py @@ -1,10 +1,10 @@ import logging import sys -import src.logging.audit -import src.logging.formatters as formatters -import src.logging.pii as pii -from src.util.env_config import PydanticBaseEnvConfig +import api.logging.audit +import api.logging.formatters as formatters +import api.logging.pii as pii +from api.util.env_config import PydanticBaseEnvConfig logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def configure_logging() -> logging.Logger: logging.root.setLevel(config.level) if config.enable_audit: - src.logging.audit.init() + api.logging.audit.init() # Configure loggers for third party packages logging.getLogger("alembic").setLevel(logging.INFO) diff --git a/app/src/logging/decodelog.py b/app/api/logging/decodelog.py similarity index 99% rename from app/src/logging/decodelog.py rename to app/api/logging/decodelog.py index c8211f96..3949d49a 100644 --- a/app/src/logging/decodelog.py +++ b/app/api/logging/decodelog.py @@ -94,7 +94,7 @@ def colorize(text: str, color: str) -> str: def color_for_name(name: str) -> str: - if name.startswith("src"): + if name.startswith("api"): return GREEN elif name.startswith("sqlalchemy"): return ORANGE diff --git a/app/src/logging/flask_logger.py b/app/api/logging/flask_logger.py similarity index 95% rename from app/src/logging/flask_logger.py rename to app/api/logging/flask_logger.py index 73e09cd3..ea517916 100644 --- a/app/src/logging/flask_logger.py +++ b/app/api/logging/flask_logger.py @@ -9,7 +9,7 @@ non-404 request. Usage: - import src.logging.flask_logger as flask_logger + import api.logging.flask_logger as flask_logger logger = logging.getLogger(__name__) app = create_app() @@ -35,7 +35,7 @@ def init_app(app_logger: logging.Logger, app: flask.Flask) -> None: Also configures the app to log every non-404 request using the given logger. Usage: - import src.logging.flask_logger as flask_logger + import api.logging.flask_logger as flask_logger logger = logging.getLogger(__name__) app = create_app() @@ -77,7 +77,7 @@ def _log_start_request() -> None: """Log the start of a request. This function handles the Flask's before_request event. - See https://tedboy.github.io/flask/interface_src.application_object.html#flask.Flask.before_request + See https://tedboy.github.io/flask/interface_api.application_object.html#flask.Flask.before_request Additional info about the request will be in the `extra` field added by `_add_request_context_info_to_log_record` @@ -89,7 +89,7 @@ def _log_end_request(response: flask.Response) -> flask.Response: """Log the end of a request. This function handles the Flask's after_request event. - See https://tedboy.github.io/flask/interface_src.application_object.html#flask.Flask.after_request + See https://tedboy.github.io/flask/interface_api.application_object.html#flask.Flask.after_request Additional info about the request will be in the `extra` field added by `_add_request_context_info_to_log_record` diff --git a/app/src/logging/formatters.py b/app/api/logging/formatters.py similarity index 97% rename from app/src/logging/formatters.py rename to app/api/logging/formatters.py index abd1d00d..e93cbe32 100644 --- a/app/src/logging/formatters.py +++ b/app/api/logging/formatters.py @@ -11,7 +11,7 @@ import logging from datetime import datetime -import src.logging.decodelog as decodelog +import api.logging.decodelog as decodelog class JsonFormatter(logging.Formatter): diff --git a/app/src/logging/pii.py b/app/api/logging/pii.py similarity index 97% rename from app/src/logging/pii.py rename to app/api/logging/pii.py index 8cce58f9..67098618 100644 --- a/app/src/logging/pii.py +++ b/app/api/logging/pii.py @@ -8,7 +8,7 @@ Example: import logging - import src.logging.pii as pii + import api.logging.pii as pii handler = logging.StreamHandler() handler.addFilter(pii.mask_pii) @@ -22,7 +22,7 @@ Example: import logging - import src.logging.pii as pii + import api.logging.pii as pii logger = logging.getLogger(__name__) logger.addFilter(pii.mask_pii) diff --git a/app/src/services/users/__init__.py b/app/api/services/users/__init__.py similarity index 100% rename from app/src/services/users/__init__.py rename to app/api/services/users/__init__.py diff --git a/app/src/services/users/create_user.py b/app/api/services/users/create_user.py similarity index 90% rename from app/src/services/users/create_user.py rename to app/api/services/users/create_user.py index 9721594a..096f9621 100644 --- a/app/src/services/users/create_user.py +++ b/app/api/services/users/create_user.py @@ -1,9 +1,9 @@ from datetime import date from typing import TypedDict -from src.adapters.db import Session -from src.db.models import user_models -from src.db.models.user_models import Role, User +from api.adapters.db import Session +from api.db.models import user_models +from api.db.models.user_models import Role, User class RoleParams(TypedDict): diff --git a/app/src/services/users/create_user_csv.py b/app/api/services/users/create_user_csv.py similarity index 96% rename from app/src/services/users/create_user_csv.py rename to app/api/services/users/create_user_csv.py index 959012f6..761d288e 100644 --- a/app/src/services/users/create_user_csv.py +++ b/app/api/services/users/create_user_csv.py @@ -4,8 +4,8 @@ from smart_open import open as smart_open -import src.adapters.db as db -from src.db.models.user_models import User +import api.adapters.db as db +from api.db.models.user_models import User logger = logging.getLogger(__name__) diff --git a/app/src/services/users/get_user.py b/app/api/services/users/get_user.py similarity index 91% rename from app/src/services/users/get_user.py rename to app/api/services/users/get_user.py index 4af4c61c..2947ad69 100644 --- a/app/src/services/users/get_user.py +++ b/app/api/services/users/get_user.py @@ -1,8 +1,8 @@ import apiflask from sqlalchemy import orm -from src.adapters.db import Session -from src.db.models.user_models import User +from api.adapters.db import Session +from api.db.models.user_models import User # TODO: separate controller and service concerns diff --git a/app/src/services/users/patch_user.py b/app/api/services/users/patch_user.py similarity index 94% rename from app/src/services/users/patch_user.py rename to app/api/services/users/patch_user.py index bb23a22d..c71653e4 100644 --- a/app/src/services/users/patch_user.py +++ b/app/api/services/users/patch_user.py @@ -6,9 +6,9 @@ import apiflask from sqlalchemy import orm -from src.adapters.db import Session -from src.db.models.user_models import Role, User -from src.services.users.create_user import RoleParams +from api.adapters.db import Session +from api.db.models.user_models import Role, User +from api.services.users.create_user import RoleParams class PatchUserParams(TypedDict, total=False): diff --git a/app/src/util/__init__.py b/app/api/util/__init__.py similarity index 100% rename from app/src/util/__init__.py rename to app/api/util/__init__.py diff --git a/app/src/util/collections/__init__.py b/app/api/util/collections/__init__.py similarity index 100% rename from app/src/util/collections/__init__.py rename to app/api/util/collections/__init__.py diff --git a/app/src/util/collections/dict.py b/app/api/util/collections/dict.py similarity index 100% rename from app/src/util/collections/dict.py rename to app/api/util/collections/dict.py diff --git a/app/src/util/datetime_util.py b/app/api/util/datetime_util.py similarity index 100% rename from app/src/util/datetime_util.py rename to app/api/util/datetime_util.py diff --git a/app/src/util/env_config.py b/app/api/util/env_config.py similarity index 78% rename from app/src/util/env_config.py rename to app/api/util/env_config.py index cd9c4dc3..c556c465 100644 --- a/app/src/util/env_config.py +++ b/app/api/util/env_config.py @@ -2,10 +2,10 @@ from pydantic import BaseSettings -import src +import api env_file = os.path.join( - os.path.dirname(os.path.dirname(src.__file__)), + os.path.dirname(os.path.dirname(api.__file__)), "config", "%s.env" % os.getenv("ENVIRONMENT", "local"), ) diff --git a/app/src/util/file_util.py b/app/api/util/file_util.py similarity index 100% rename from app/src/util/file_util.py rename to app/api/util/file_util.py diff --git a/app/src/util/local.py b/app/api/util/local.py similarity index 100% rename from app/src/util/local.py rename to app/api/util/local.py diff --git a/app/src/util/string_utils.py b/app/api/util/string_utils.py similarity index 100% rename from app/src/util/string_utils.py rename to app/api/util/string_utils.py diff --git a/app/local.env b/app/local.env index 04c30a3a..63f135e9 100644 --- a/app/local.env +++ b/app/local.env @@ -1,6 +1,6 @@ # Local environment variables # Used by docker-compose and it can be loaded -# by calling load_local_env_vars() from app/src/util/local.py +# by calling load_local_env_vars() from app/api/util/local.py ENVIRONMENT=local PORT=8080 @@ -15,7 +15,7 @@ PYTHONPATH=/app/ # commands that can run in or out # of the Docker container - defaults to outside -FLASK_APP=src.app:create_app +FLASK_APP=api.app:create_app ############################ # Logging diff --git a/app/pyproject.toml b/app/pyproject.toml index 7e4af099..eb85e2cd 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -2,7 +2,7 @@ name = "template-application-flask" version = "0.1.0" description = "A template flask API for building ontop of" -packages = [{ include = "src" }] +packages = [{ include = "api" }] authors = ["Nava Engineering "] [tool.poetry.dependencies] @@ -44,9 +44,9 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -db-migrate-up = "src.db.migrations.run:up" -db-migrate-down = "src.db.migrations.run:down" -db-migrate-down-all = "src.db.migrations.run:downall" +db-migrate-up = "api.db.migrations.run:up" +db-migrate-down = "api.db.migrations.run:down" +db-migrate-down-all = "api.db.migrations.run:downall" [tool.black] line-length = 100 @@ -85,14 +85,14 @@ plugins = ["sqlalchemy.ext.mypy.plugin"] [tool.bandit] # Ignore audit logging test file since test audit logging requires a lot of operations that trigger bandit warnings -exclude_dirs = ["./tests/src/logging/test_audit.py"] +exclude_dirs = ["./tests/api/logging/test_audit.py"] [[tool.mypy.overrides]] # Migrations are generated without "-> None" # for the returns. Rather than require manually # fixing this for every migration generated, # disable the check for that folder. -module = "src.db.migrations.versions.*" +module = "api.db.migrations.versions.*" disallow_untyped_defs = false [tool.pytest.ini_options] @@ -108,5 +108,5 @@ markers = [ "audit: mark a test as a security audit log test, to be run isolated from other tests"] [tool.coverage.run] -omit = ["src/db/migrations/*.py"] +omit = ["api/db/migrations/*.py"] diff --git a/app/tests/src/__init__.py b/app/tests/api/__init__.py similarity index 100% rename from app/tests/src/__init__.py rename to app/tests/api/__init__.py diff --git a/app/tests/src/adapters/__init__.py b/app/tests/api/adapters/__init__.py similarity index 100% rename from app/tests/src/adapters/__init__.py rename to app/tests/api/adapters/__init__.py diff --git a/app/tests/src/adapters/db/test_db.py b/app/tests/api/adapters/db/test_db.py similarity index 96% rename from app/tests/src/adapters/db/test_db.py rename to app/tests/api/adapters/db/test_db.py index 77f2c8f8..de68e044 100644 --- a/app/tests/src/adapters/db/test_db.py +++ b/app/tests/api/adapters/db/test_db.py @@ -4,9 +4,9 @@ import pytest from sqlalchemy import text -import src.adapters.db as db -from src.adapters.db.client import get_connection_parameters, make_connection_uri, verify_ssl -from src.adapters.db.config import DbConfig, get_db_config +import api.adapters.db as db +from api.adapters.db.client import get_connection_parameters, make_connection_uri, verify_ssl +from api.adapters.db.config import DbConfig, get_db_config class DummyConnectionInfo: diff --git a/app/tests/src/adapters/db/test_flask_db.py b/app/tests/api/adapters/db/test_flask_db.py similarity index 93% rename from app/tests/src/adapters/db/test_flask_db.py rename to app/tests/api/adapters/db/test_flask_db.py index 03c5ccd9..dfca9964 100644 --- a/app/tests/src/adapters/db/test_flask_db.py +++ b/app/tests/api/adapters/db/test_flask_db.py @@ -2,8 +2,8 @@ from flask import Flask, current_app from sqlalchemy import text -import src.adapters.db as db -import src.adapters.db.flask_db as flask_db +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db # Define an isolated example Flask app fixture specific to this test module diff --git a/app/tests/src/auth/test_api_key_auth.py b/app/tests/api/auth/test_api_key_auth.py similarity index 93% rename from app/tests/src/auth/test_api_key_auth.py rename to app/tests/api/auth/test_api_key_auth.py index 152cc63f..c3434d7b 100644 --- a/app/tests/src/auth/test_api_key_auth.py +++ b/app/tests/api/auth/test_api_key_auth.py @@ -2,7 +2,7 @@ from apiflask import HTTPError from flask import g -from src.auth.api_key_auth import API_AUTH_USER, verify_token +from api.auth.api_key_auth import API_AUTH_USER, verify_token def test_verify_token_success(app, api_auth_token): diff --git a/app/tests/src/db/__init__.py b/app/tests/api/db/__init__.py similarity index 100% rename from app/tests/src/db/__init__.py rename to app/tests/api/db/__init__.py diff --git a/app/tests/src/db/models/__init__.py b/app/tests/api/db/models/__init__.py similarity index 100% rename from app/tests/src/db/models/__init__.py rename to app/tests/api/db/models/__init__.py diff --git a/app/tests/src/db/models/factories.py b/app/tests/api/db/models/factories.py similarity index 93% rename from app/tests/src/db/models/factories.py rename to app/tests/api/db/models/factories.py index 7edf8118..c822f680 100644 --- a/app/tests/src/db/models/factories.py +++ b/app/tests/api/db/models/factories.py @@ -15,9 +15,9 @@ import faker from sqlalchemy.orm import scoped_session -import src.adapters.db as db -import src.db.models.user_models as user_models -import src.util.datetime_util as datetime_util +import api.adapters.db as db +import api.db.models.user_models as user_models +import api.util.datetime_util as datetime_util _db_session: Optional[db.Session] = None @@ -67,7 +67,7 @@ class Meta: model = user_models.Role user_id = factory.LazyAttribute(lambda u: u.user.id) - user = factory.SubFactory("tests.src.db.models.factories.UserFactory", roles=[]) + user = factory.SubFactory("tests.api.db.models.factories.UserFactory", roles=[]) type = factory.Iterator([r.value for r in user_models.RoleType]) diff --git a/app/tests/src/db/models/test_factories.py b/app/tests/api/db/models/test_factories.py similarity index 96% rename from app/tests/src/db/models/test_factories.py rename to app/tests/api/db/models/test_factories.py index b3370a3f..79807224 100644 --- a/app/tests/src/db/models/test_factories.py +++ b/app/tests/api/db/models/test_factories.py @@ -2,8 +2,8 @@ import pytest -from src.db.models.user_models import User -from tests.src.db.models.factories import RoleFactory, UserFactory +from api.db.models.user_models import User +from tests.api.db.models.factories import RoleFactory, UserFactory user_params = { "first_name": "Alvin", diff --git a/app/tests/src/db/test_migrations.py b/app/tests/api/db/test_migrations.py similarity index 96% rename from app/tests/src/db/test_migrations.py rename to app/tests/api/db/test_migrations.py index 9a54ae03..0f286631 100644 --- a/app/tests/src/db/test_migrations.py +++ b/app/tests/api/db/test_migrations.py @@ -6,7 +6,7 @@ from alembic.script.revision import MultipleHeads from alembic.util.exc import CommandError -from src.db.migrations.run import alembic_cfg +from api.db.migrations.run import alembic_cfg def test_only_single_head_revision_in_migrations(): diff --git a/app/tests/src/logging/__init__.py b/app/tests/api/logging/__init__.py similarity index 100% rename from app/tests/src/logging/__init__.py rename to app/tests/api/logging/__init__.py diff --git a/app/tests/src/logging/test_audit.py b/app/tests/api/logging/test_audit.py similarity index 99% rename from app/tests/src/logging/test_audit.py rename to app/tests/api/logging/test_audit.py index 70213b5e..874c9a9b 100644 --- a/app/tests/src/logging/test_audit.py +++ b/app/tests/api/logging/test_audit.py @@ -1,5 +1,5 @@ # -# Tests for src.logging.audit. +# Tests for api.logging.audit. # import logging @@ -16,7 +16,7 @@ import pytest -import src.logging.audit as audit +import api.logging.audit as audit # Do not run these tests alongside the rest of the test suite since # this tests adds an audit hook that interfere with other tests, diff --git a/app/tests/src/logging/test_flask_logger.py b/app/tests/api/logging/test_flask_logger.py similarity index 95% rename from app/tests/src/logging/test_flask_logger.py rename to app/tests/api/logging/test_flask_logger.py index 7a6bf85c..c26250b8 100644 --- a/app/tests/src/logging/test_flask_logger.py +++ b/app/tests/api/logging/test_flask_logger.py @@ -4,13 +4,13 @@ import pytest from flask import Flask -import src.logging.flask_logger as flask_logger +import api.logging.flask_logger as flask_logger from tests.lib.assertions import assert_dict_contains @pytest.fixture def logger(): - logger = logging.getLogger("src") + logger = logging.getLogger("api") logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) return logger @@ -22,7 +22,7 @@ def app(logger): @app.get("/hello/") def hello(name): - logging.getLogger("src.hello").info(f"hello, {name}!") + logging.getLogger("api.hello").info(f"hello, {name}!") return "ok" flask_logger.init_app(logger, app) diff --git a/app/tests/src/logging/test_formatters.py b/app/tests/api/logging/test_formatters.py similarity index 97% rename from app/tests/src/logging/test_formatters.py rename to app/tests/api/logging/test_formatters.py index 28b84d57..ecebe241 100644 --- a/app/tests/src/logging/test_formatters.py +++ b/app/tests/api/logging/test_formatters.py @@ -4,7 +4,7 @@ import pytest -import src.logging.formatters as formatters +import api.logging.formatters as formatters from tests.lib.assertions import assert_dict_contains diff --git a/app/tests/src/logging/test_logging.py b/app/tests/api/logging/test_logging.py similarity index 95% rename from app/tests/src/logging/test_logging.py rename to app/tests/api/logging/test_logging.py index cb5ceba3..53d5a078 100644 --- a/app/tests/src/logging/test_logging.py +++ b/app/tests/api/logging/test_logging.py @@ -3,8 +3,8 @@ import pytest -import src.logging -import src.logging.formatters as formatters +import api.logging +import api.logging.formatters as formatters def _init_test_logger( @@ -12,7 +12,7 @@ def _init_test_logger( ): caplog.set_level(logging.DEBUG) monkeypatch.setenv("LOG_FORMAT", log_format) - src.logging.init("test_logging") + api.logging.init("test_logging") @pytest.fixture diff --git a/app/tests/src/logging/test_pii.py b/app/tests/api/logging/test_pii.py similarity index 96% rename from app/tests/src/logging/test_pii.py rename to app/tests/api/logging/test_pii.py index 8b33652d..67c4eedf 100644 --- a/app/tests/src/logging/test_pii.py +++ b/app/tests/api/logging/test_pii.py @@ -1,6 +1,6 @@ import pytest -import src.logging.pii as pii +import api.logging.pii as pii @pytest.mark.parametrize( diff --git a/app/tests/src/route/__init__.py b/app/tests/api/route/__init__.py similarity index 100% rename from app/tests/src/route/__init__.py rename to app/tests/api/route/__init__.py diff --git a/app/tests/src/route/test_healthcheck.py b/app/tests/api/route/test_healthcheck.py similarity index 94% rename from app/tests/src/route/test_healthcheck.py rename to app/tests/api/route/test_healthcheck.py index 1fe7fd53..32524e18 100644 --- a/app/tests/src/route/test_healthcheck.py +++ b/app/tests/api/route/test_healthcheck.py @@ -1,4 +1,4 @@ -import src.adapters.db as db +import api.adapters.db as db def test_get_healthcheck_200(client): diff --git a/app/tests/src/route/test_user_route.py b/app/tests/api/route/test_user_route.py similarity index 99% rename from app/tests/src/route/test_user_route.py rename to app/tests/api/route/test_user_route.py index f8b38efd..00a8b80a 100644 --- a/app/tests/src/route/test_user_route.py +++ b/app/tests/api/route/test_user_route.py @@ -3,7 +3,7 @@ import faker import pytest -from tests.src.util.parametrize_utils import powerset +from tests.api.util.parametrize_utils import powerset fake = faker.Faker() diff --git a/app/tests/src/scripts/__init__.py b/app/tests/api/scripts/__init__.py similarity index 100% rename from app/tests/src/scripts/__init__.py rename to app/tests/api/scripts/__init__.py diff --git a/app/tests/src/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py similarity index 92% rename from app/tests/src/scripts/test_create_user_csv.py rename to app/tests/api/scripts/test_create_user_csv.py index a938eb79..ea615daa 100644 --- a/app/tests/src/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -7,13 +7,13 @@ from pytest_lazyfixture import lazy_fixture from smart_open import open as smart_open -import src.adapters.db as db -import src.app as app_entry -import tests.src.db.models.factories as factories -from src.db import models -from src.db.models.user_models import User +import api.adapters.db as db +import api.app as app_entry +import tests.api.db.models.factories as factories +from api.db import models +from api.db.models.user_models import User +from tests.api.db.models.factories import UserFactory from tests.lib import db_testing -from tests.src.db.models.factories import UserFactory @pytest.fixture diff --git a/app/tests/src/scripts/test_create_user_csv_expected.csv b/app/tests/api/scripts/test_create_user_csv_expected.csv similarity index 100% rename from app/tests/src/scripts/test_create_user_csv_expected.csv rename to app/tests/api/scripts/test_create_user_csv_expected.csv diff --git a/app/tests/src/util/__init__.py b/app/tests/api/util/__init__.py similarity index 100% rename from app/tests/src/util/__init__.py rename to app/tests/api/util/__init__.py diff --git a/app/tests/src/util/collections/__init__.py b/app/tests/api/util/collections/__init__.py similarity index 100% rename from app/tests/src/util/collections/__init__.py rename to app/tests/api/util/collections/__init__.py diff --git a/app/tests/src/util/collections/test_dict.py b/app/tests/api/util/collections/test_dict.py similarity index 92% rename from app/tests/src/util/collections/test_dict.py rename to app/tests/api/util/collections/test_dict.py index 242b4dc6..1a8e98bb 100644 --- a/app/tests/src/util/collections/test_dict.py +++ b/app/tests/api/util/collections/test_dict.py @@ -1,8 +1,8 @@ # -# Unit tests for src.util.collections.dict. +# Unit tests for api.util.collections.dict. # -import src.util.collections.dict as dict_util +import api.util.collections.dict as dict_util def test_least_recently_used_dict(): diff --git a/app/tests/src/util/parametrize_utils.py b/app/tests/api/util/parametrize_utils.py similarity index 100% rename from app/tests/src/util/parametrize_utils.py rename to app/tests/api/util/parametrize_utils.py diff --git a/app/tests/src/util/test_datetime_util.py b/app/tests/api/util/test_datetime_util.py similarity index 97% rename from app/tests/src/util/test_datetime_util.py rename to app/tests/api/util/test_datetime_util.py index d6bf02d9..9a86200c 100644 --- a/app/tests/src/util/test_datetime_util.py +++ b/app/tests/api/util/test_datetime_util.py @@ -3,7 +3,7 @@ import pytest import pytz -from src.util.datetime_util import adjust_timezone +from api.util.datetime_util import adjust_timezone @pytest.mark.parametrize( diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 66a8e1e7..8bd26567 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -8,11 +8,11 @@ import pytest import sqlalchemy -import src.adapters.db as db -import src.app as app_entry -import tests.src.db.models.factories as factories -from src.db import models -from src.util.local import load_local_env_vars +import api.adapters.db as db +import api.app as app_entry +import tests.api.db.models.factories as factories +from api.db import models +from api.util.local import load_local_env_vars from tests.lib import db_testing logger = logging.getLogger(__name__) diff --git a/app/tests/lib/db_testing.py b/app/tests/lib/db_testing.py index 05a8fef5..911d113e 100644 --- a/app/tests/lib/db_testing.py +++ b/app/tests/lib/db_testing.py @@ -3,8 +3,8 @@ import logging import uuid -import src.adapters.db as db -from src.adapters.db.config import get_db_config +import api.adapters.db as db +from api.adapters.db.config import get_db_config logger = logging.getLogger(__name__) diff --git a/docs/app/README.md b/docs/app/README.md index 43bcf83d..ba0f0742 100644 --- a/docs/app/README.md +++ b/docs/app/README.md @@ -12,7 +12,7 @@ This is the API layer. It includes a few separate components: ```text root ├── app -│ └── src +│ └── api │ └── auth Authentication code for API │ └── db │ └── models DB model definitions @@ -68,7 +68,7 @@ Running in the native/local approach may require additional packages to be insta Most configuration options are managed by environment variables. -Environment variables for local development are stored in the [local.env](/app/local.env) file. This file is automatically loaded when running. If running within Docker, this file is specified as an `env_file` in the [docker-compose](/docker-compose.yml) file, and loaded [by a script](/app/src/util/local.py) automatically when running most other components outside the container. +Environment variables for local development are stored in the [local.env](/app/local.env) file. This file is automatically loaded when running. If running within Docker, this file is specified as an `env_file` in the [docker-compose](/docker-compose.yml) file, and loaded [by a script](/app/api/util/local.py) automatically when running most other components outside the container. Any environment variables specified directly in the [docker-compose](/docker-compose.yml) file will take precedent over those specified in the [local.env](/app/local.env) file. diff --git a/docs/app/api-details.md b/docs/app/api-details.md index 9606e2fa..893199c8 100644 --- a/docs/app/api-details.md +++ b/docs/app/api-details.md @@ -4,7 +4,7 @@ See [Technical Overview](./technical-overview.md) for details on the technologie Each endpoint is configured in the [openapi.yml](/app/openapi.yml) file which provides basic request validation. Each endpoint specifies an `operationId` that maps to a function defined in the code that will handle the request. -To make handling a request easier, an [ApiContext](/app/src/util/api_context.py) exists which will fetch the DB session, request body, and current user. This can be used like so: +To make handling a request easier, an [ApiContext](/app/api/util/api_context.py) exists which will fetch the DB session, request body, and current user. This can be used like so: ```py def example_post() -> flask.Response: with api_context_manager() as api_context: @@ -39,7 +39,7 @@ All model schemas defined can be found at the bottom of the UI. # Routes ## Health Check -[GET /v1/healthcheck](/app/src/route/healthcheck.py) is an endpoint for checking the health of the service. It verifies that the database is reachable, and that the API service itself is up and running. +[GET /v1/healthcheck](/app/api/route/healthcheck.py) is an endpoint for checking the health of the service. It verifies that the database is reachable, and that the API service itself is up and running. Note this endpoint explicitly does not require authorization so it can be integrated with any automated monitoring you may build. diff --git a/docs/app/database/database-access-management.md b/docs/app/database/database-access-management.md index 60faa27f..267a3987 100644 --- a/docs/app/database/database-access-management.md +++ b/docs/app/database/database-access-management.md @@ -4,7 +4,7 @@ This document describes the best practices and patterns for how the application ## Client Initialization and Configuration -The database client is initialized when the application starts (see [src/\_\_main\_\_.py](../../../app/src/__main__.py). As the database engine that is used to create acquire connections to the database is initialized using the database configuration defined in [db_config.py](../../../app/src/db/db_config.py), which is configured through environment variables. The initialized database client is then stored on the Flask app's [\`extensions\` dictionary](https://flask.palletsprojects.com/en/2.2.x/src/#flask.Flask.extensions) to be used throughout the lifetime of the application. +The database client is initialized when the application starts (see [api/\_\_main\_\_.py](../../../app/api/__main__.py). As the database engine that is used to create acquire connections to the database is initialized using the database configuration defined in [db_config.py](../../../app/api/db/db_config.py), which is configured through environment variables. The initialized database client is then stored on the Flask app's [\`extensions\` dictionary](https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.extensions) to be used throughout the lifetime of the application. ## Session Management @@ -16,7 +16,7 @@ For example, **do this** ### right way ### from flask import current_app -import src.adapters.db as db +import api.adapters.db as db def some_service_func(session: db.Session) with db_session.begin(): # start transaction @@ -35,7 +35,7 @@ and **don't do this** ### wrong way ### from flask import current_app -import src.adapters.db as db +import api.adapters.db as db def some_service_func() db_client = db.get_db(current_app) diff --git a/docs/app/database/database-migrations.md b/docs/app/database/database-migrations.md index 31f89804..ec16929b 100644 --- a/docs/app/database/database-migrations.md +++ b/docs/app/database/database-migrations.md @@ -31,7 +31,7 @@ $ make db-upgrade
Example: Adding a new column to an existing table: -1. Manually update the database models with the changes ([example_models.py](/app/src/db/models/example_models.py) in this example) +1. Manually update the database models with the changes ([example_models.py](/app/api/db/models/example_models.py) in this example) ```python class ExampleTable(Base): ... diff --git a/docs/app/database/database-testing.md b/docs/app/database/database-testing.md index 4b52b923..1ccf9ff4 100644 --- a/docs/app/database/database-testing.md +++ b/docs/app/database/database-testing.md @@ -4,7 +4,7 @@ This document describes how the database is managed in the test suite. ## Test Schema -The test suite creates a new PostgreSQL database schema separate from the `public` schema that is used by the application outside of testing. This schema persists throughout the testing session is dropped at the end of the test run. The schema is created by the `db` fixture in [conftest.py](../../../app/tests/conftest.py). The fixture also creates and returns an initialized instance of the [db.DBClient](../../../app/src/db/__init__.py) that can be used to connect to the created schema. +The test suite creates a new PostgreSQL database schema separate from the `public` schema that is used by the application outside of testing. This schema persists throughout the testing session is dropped at the end of the test run. The schema is created by the `db` fixture in [conftest.py](../../../app/tests/conftest.py). The fixture also creates and returns an initialized instance of the [db.DBClient](../../../app/api/db/__init__.py) that can be used to connect to the created schema. Note that [PostgreSQL schemas](https://www.postgresql.org/docs/current/ddl-schemas.html) are entirely different concepts from [Schema objects in OpenAPI specification](https://swagger.io/docs/specification/data-models/). diff --git a/docs/app/formatting-and-linting.md b/docs/app/formatting-and-linting.md index bce9f557..b93b7919 100644 --- a/docs/app/formatting-and-linting.md +++ b/docs/app/formatting-and-linting.md @@ -4,7 +4,7 @@ Run `make format` to run all of the formatters. -When we run migrations via alembic, we autorun the formatters on the generated files. See [alembic.ini](/app/src/db/migrations/alembic.ini) for configuration. +When we run migrations via alembic, we autorun the formatters on the generated files. See [alembic.ini](/app/api/db/migrations/alembic.ini) for configuration. ### Isort [isort](https://pycqa.github.io/isort/) is used to sort our Python imports. Configuration options can be found in [pyproject.toml - tool.isort](/app/pyproject.toml) diff --git a/docs/app/monitoring-and-observability/logging-configuration.md b/docs/app/monitoring-and-observability/logging-configuration.md index 993b048d..1a62e37b 100644 --- a/docs/app/monitoring-and-observability/logging-configuration.md +++ b/docs/app/monitoring-and-observability/logging-configuration.md @@ -2,7 +2,7 @@ ## Overview -This document describes how logging is configured in the application. The logging functionality is defined in the [src.logging](../../../app/src/logging/) package and leverages Python's built-in [logging](https://docs.python.org/3/library/logging.html) framework. +This document describes how logging is configured in the application. The logging functionality is defined in the [api.logging](../../../app/api/logging/) package and leverages Python's built-in [logging](https://docs.python.org/3/library/logging.html) framework. ## Formatting @@ -12,7 +12,7 @@ We have two separate ways of formatting the logs which are controlled by the `LO ```json { - "name": "src.api.healthcheck", + "name": "api.api.healthcheck", "levelname": "INFO", "funcName": "healthcheck_get", "created": "1663261542.0465896", @@ -33,15 +33,15 @@ We have two separate ways of formatting the logs which are controlled by the `LO ## Logging Extra Data in a Request -The [src.logging.flask_logger](../../../app/src/logging/flask_logger.py) module adds logging functionality to Flask applications. It automatically adds useful data from the Flask request object to logs, logs the start and end of requests, and provides a mechanism for developers to dynamically add extra data to all subsequent logs for the current request. +The [api.logging.flask_logger](../../../app/api/logging/flask_logger.py) module adds logging functionality to Flask applications. It automatically adds useful data from the Flask request object to logs, logs the start and end of requests, and provides a mechanism for developers to dynamically add extra data to all subsequent logs for the current request. ## PII Masking -The src.logging.pii](../../../app/src/logging/pii.py) module defines a filter that applies to all logs that automatically masks data fields that look like social security numbers. +The api.logging.pii](../../../app/api/logging/pii.py) module defines a filter that applies to all logs that automatically masks data fields that look like social security numbers. ## Audit Logging -* The [src.logging.audit](../../../app/src/logging/audit.py) module defines a low level audit hook that logs events that may be of interest from a security point of view, such as dynamic code execution and network requests. +* The [api.logging.audit](../../../app/api/logging/audit.py) module defines a low level audit hook that logs events that may be of interest from a security point of view, such as dynamic code execution and network requests. ## Additional Reading diff --git a/docs/app/writing-tests.md b/docs/app/writing-tests.md index 69f31bc9..e94626f6 100644 --- a/docs/app/writing-tests.md +++ b/docs/app/writing-tests.md @@ -16,7 +16,7 @@ For this project specifically: - All tests live under `app/tests/` - Under `tests/`, the organization mirrors the source code structure - - The tests for `app/src/route/` are found at `app/test/api/route/` + - The tests for `app/api/route/` are found at `app/test/api/route/` - Create `__init__.py` files for each directory. This helps [avoid name conflicts when pytest is resolving tests](https://docs.pytest.org/en/stable/goodpractices.html#tests-outside-application-code). @@ -66,7 +66,7 @@ They can be imported into tests from the path `tests.helpers`, for example, To facilitate easier setup of test data, most database models have factories via [factory_boy](https://factoryboy.readthedocs.io/) in -`app/src/db/models/factories.py`. +`app/api/db/models/factories.py`. There are a few different ways of [using the factories](https://factoryboy.readthedocs.io/en/stable/#using-factories), termed @@ -92,4 +92,4 @@ FooFactory.build(foo_id=5, name="Bar") ``` would set `foo_id=5` and `name="Bar"` on the generated model, while all other -attributes would use what's configured on the factory class. +attributes would use what's configured on the factory class. \ No newline at end of file From aeff01a645c05fb17f4152be08c3a5982f60f0a2 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 09:33:33 -0800 Subject: [PATCH 37/51] Remove rollback logic from test --- app/tests/conftest.py | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 8bd26567..d6cd34b3 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -79,32 +79,17 @@ def empty_schema(monkeypatch) -> db.DBClient: @pytest.fixture -def test_db_session(db_client: db.DBClient) -> db.Session: +def db_session(db_client: db.DBClient) -> db.Session: # Based on https://docs.sqlalchemy.org/en/13/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites - with db_client.get_connection() as connection: - trans = connection.begin() - - # Rather than call db.get_session() to create a new session with a new connection, - # create a session bound to the existing connection that has a transaction manually start. - # This allows the transaction to be rolled back after the test completes. - with db.Session(bind=connection, autocommit=False, expire_on_commit=False) as session: - session.begin_nested() - - @sqlalchemy.event.listens_for(session, "after_transaction_end") - def restart_savepoint(session, transaction): - if transaction.nested and not transaction._parent.nested: - session.begin_nested() - - yield session - - trans.rollback() + with db_client.get_session() as session: + yield session @pytest.fixture -def factories_db_session(monkeypatch, test_db_session) -> db.Session: - monkeypatch.setattr(factories, "_db_session", test_db_session) - logger.info("set factories db_session to %s", test_db_session) - return test_db_session +def factories_db_session(monkeypatch, db_session) -> db.Session: + monkeypatch.setattr(factories, "_db_session", db_session) + logger.info("set factories db_session to %s", db_session) + return db_session #################### From 55c233905c1e06a24190ef34b84175eda4f7ff51 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:17:23 -0800 Subject: [PATCH 38/51] Add migration to cascade on delete --- .../versions/2023_02_21_cascade_on_delete.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py diff --git a/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py b/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py new file mode 100644 index 00000000..c793d595 --- /dev/null +++ b/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py @@ -0,0 +1,31 @@ +"""cascade on delete + +Revision ID: 9fe657340f70 +Revises: 4ff1160282d1 +Create Date: 2023-02-21 18:16:56.052679 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "9fe657340f70" +down_revision = "4ff1160282d1" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("role_user_id_user_fkey", "role", type_="foreignkey") + op.create_foreign_key( + op.f("role_user_id_user_fkey"), "role", "user", ["user_id"], ["id"], ondelete="CASCADE" + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f("role_user_id_user_fkey"), "role", type_="foreignkey") + op.create_foreign_key("role_user_id_user_fkey", "role", "user", ["user_id"], ["id"]) + # ### end Alembic commands ### From 23a1a759392a1e37c5766bd467ab65c1399abf34 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:18:33 -0800 Subject: [PATCH 39/51] Truncate user table in create csv test --- app/api/db/models/user_models.py | 6 ++- app/tests/api/scripts/test_create_user_csv.py | 44 ++++--------------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/app/api/db/models/user_models.py b/app/api/db/models/user_models.py index 47341160..6fca730e 100644 --- a/app/api/db/models/user_models.py +++ b/app/api/db/models/user_models.py @@ -28,13 +28,15 @@ class User(Base, IdMixin, TimestampMixin): date_of_birth: date = Column(Date, nullable=False) is_active: bool = Column(Boolean, nullable=False) - roles: list["Role"] = relationship("Role", back_populates="user", order_by="Role.type") + roles: list["Role"] = relationship( + "Role", back_populates="user", cascade="all, delete", order_by="Role.type" + ) class Role(Base, TimestampMixin): __tablename__ = "role" user_id: Mapped[UUID] = Column( - postgresql.UUID(as_uuid=True), ForeignKey("user.id"), primary_key=True + postgresql.UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), primary_key=True ) # Set native_enum=False to use store enum values as VARCHAR/TEXT diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index ea615daa..319b0135 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -2,54 +2,25 @@ import os.path as path import re +import api.adapters.db as db import flask.testing import pytest +from api.db.models.user_models import User from pytest_lazyfixture import lazy_fixture from smart_open import open as smart_open -import api.adapters.db as db -import api.app as app_entry import tests.api.db.models.factories as factories -from api.db import models -from api.db.models.user_models import User from tests.api.db.models.factories import UserFactory from tests.lib import db_testing @pytest.fixture -def isolated_db(monkeypatch) -> db.DBClient: - """ - Creates an isolated database for the test function. - - Creates a new empty PostgreSQL schema, creates all tables in the new schema - using SQLAlchemy, then returns a db.DB instance that can be used to - get connections or sessions to this database schema. The schema is dropped - after the test function completes. - - This is similar to the db fixture except the scope of the schema is the - individual test rather the test session. - """ - - with db_testing.create_isolated_db(monkeypatch) as db: - models.metadata.create_all(bind=db.get_connection()) - yield db - - -@pytest.fixture -def cli_runner(isolated_db) -> flask.testing.CliRunner: - """Overrides cli_runner from conftest to run in an isolated db schema""" - return app_entry.create_app().test_cli_runner() - - -@pytest.fixture -def isolated_db_factories_session(monkeypatch, isolated_db: db.DBClient) -> db.Session: - with isolated_db.get_session() as session: - monkeypatch.setattr(factories, "_db_session", session) - yield session +def truncate_user(db_session: db.Session): + db_session.query(User).delete() @pytest.fixture -def prepopulated_users(isolated_db_factories_session) -> list[User]: +def prepopulated_users(factories_db_session) -> list[User]: return [ UserFactory.create(first_name="Jon", last_name="Doe", is_active=True), UserFactory.create(first_name="Jane", last_name="Doe", is_active=False), @@ -74,7 +45,10 @@ def tmp_s3_folder(mock_s3_bucket): ], ) def test_create_user_csv( - cli_runner: flask.testing.FlaskCliRunner, prepopulated_users: list[User], dir: str + truncate_user, + cli_runner: flask.testing.FlaskCliRunner, + prepopulated_users: list[User], + dir: str, ): cli_runner.invoke(args=["user", "create-csv", "--dir", dir, "--filename", "test.csv"]) output = smart_open(path.join(dir, "test.csv")).read() From ddc6313d46a5cfb62f6e697262df5e636741ffb2 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:19:40 -0800 Subject: [PATCH 40/51] Move empty_schema fixture to test_migrations --- app/tests/api/db/models/test_factories.py | 18 +++++++++++++++++- app/tests/conftest.py | 14 -------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/app/tests/api/db/models/test_factories.py b/app/tests/api/db/models/test_factories.py index 79807224..5b39432c 100644 --- a/app/tests/api/db/models/test_factories.py +++ b/app/tests/api/db/models/test_factories.py @@ -1,9 +1,25 @@ from datetime import date, datetime +import api.adapters.db as db import pytest - from api.db.models.user_models import User + from tests.api.db.models.factories import RoleFactory, UserFactory +from tests.lib import db_testing + + +@pytest.fixture +def empty_schema(monkeypatch) -> db.DBClient: + """ + Create a test schema, if it doesn't already exist, and drop it after the + test completes. + + This is similar to the db fixture but does not create any tables in the + schema. This is used by migration tests. + """ + with db_testing.create_isolated_db(monkeypatch) as db: + yield db + user_params = { "first_name": "Alvin", diff --git a/app/tests/conftest.py b/app/tests/conftest.py index d6cd34b3..a1070c93 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -6,7 +6,6 @@ import flask.testing import moto import pytest -import sqlalchemy import api.adapters.db as db import api.app as app_entry @@ -65,19 +64,6 @@ def db_client(monkeypatch_session) -> db.DBClient: yield db_client -@pytest.fixture -def empty_schema(monkeypatch) -> db.DBClient: - """ - Create a test schema, if it doesn't already exist, and drop it after the - test completes. - - This is similar to the db fixture but does not create any tables in the - schema. This is used by migration tests. - """ - with db_testing.create_isolated_db(monkeypatch) as db: - yield db - - @pytest.fixture def db_session(db_client: db.DBClient) -> db.Session: # Based on https://docs.sqlalchemy.org/en/13/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites From b83213bc4c6ddb717020c7204d4e4d10d2a6e62b Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:21:59 -0800 Subject: [PATCH 41/51] Rename factories_db_session fixture --- app/api/adapters/db/client.py | 3 +-- app/api/adapters/db/config.py | 3 +-- app/api/adapters/db/flask_db.py | 3 +-- app/api/api/healthcheck.py | 2 +- app/api/api/response.py | 3 ++- app/api/api/users/user_commands.py | 6 +++--- app/api/api/users/user_routes.py | 7 ++++--- app/api/api/users/user_schemas.py | 2 +- app/api/app.py | 10 +++++----- app/api/db/models/base.py | 3 +-- app/api/db/models/user_models.py | 3 +-- app/api/services/users/create_user_csv.py | 3 +-- app/api/services/users/get_user.py | 3 +-- app/api/services/users/patch_user.py | 3 +-- app/tests/api/adapters/db/test_db.py | 5 ++--- app/tests/api/adapters/db/test_flask_db.py | 5 ++--- app/tests/api/auth/test_api_key_auth.py | 3 +-- app/tests/api/db/models/factories.py | 11 +++++------ app/tests/api/db/models/test_factories.py | 12 ++++++------ app/tests/api/db/test_migrations.py | 1 - app/tests/api/logging/test_audit.py | 3 +-- app/tests/api/logging/test_flask_logger.py | 2 +- app/tests/api/logging/test_formatters.py | 2 +- app/tests/api/logging/test_logging.py | 3 +-- app/tests/api/logging/test_pii.py | 3 +-- app/tests/api/scripts/test_create_user_csv.py | 2 +- app/tests/api/util/test_datetime_util.py | 1 - app/tests/conftest.py | 10 +++++----- docs/app/database/database-testing.md | 2 +- 29 files changed, 52 insertions(+), 67 deletions(-) diff --git a/app/api/adapters/db/client.py b/app/api/adapters/db/client.py index 36f38eee..585bbdc2 100644 --- a/app/api/adapters/db/client.py +++ b/app/api/adapters/db/client.py @@ -15,9 +15,8 @@ import psycopg2 import sqlalchemy import sqlalchemy.pool as pool -from sqlalchemy.orm import session - from api.adapters.db.config import DbConfig, get_db_config +from sqlalchemy.orm import session # Re-export the Connection type that is returned by the get_connection() method # to be used for type hints. diff --git a/app/api/adapters/db/config.py b/app/api/adapters/db/config.py index 67475f65..a7edc9dd 100644 --- a/app/api/adapters/db/config.py +++ b/app/api/adapters/db/config.py @@ -1,9 +1,8 @@ import logging from typing import Optional -from pydantic import Field - from api.util.env_config import PydanticBaseEnvConfig +from pydantic import Field logger = logging.getLogger(__name__) diff --git a/app/api/adapters/db/flask_db.py b/app/api/adapters/db/flask_db.py index 04f486c3..788c2e73 100644 --- a/app/api/adapters/db/flask_db.py +++ b/app/api/adapters/db/flask_db.py @@ -41,10 +41,9 @@ def health(): from functools import wraps from typing import Any, Callable, Concatenate, ParamSpec, TypeVar -from flask import Flask, current_app - import api.adapters.db as db from api.adapters.db.client import DBClient +from flask import Flask, current_app _FLASK_EXTENSION_KEY = "db" diff --git a/app/api/api/healthcheck.py b/app/api/api/healthcheck.py index 4df442b2..c3d80f04 100644 --- a/app/api/api/healthcheck.py +++ b/app/api/api/healthcheck.py @@ -1,12 +1,12 @@ import logging from typing import Tuple +import api.adapters.db.flask_db as flask_db from apiflask import APIBlueprint from flask import current_app from sqlalchemy import text from werkzeug.exceptions import ServiceUnavailable -import api.adapters.db.flask_db as flask_db from api.api import response from api.api.schemas import request_schema diff --git a/app/api/api/response.py b/app/api/api/response.py index 65f6e0bb..86da4643 100644 --- a/app/api/api/response.py +++ b/app/api/api/response.py @@ -1,9 +1,10 @@ import dataclasses from typing import Optional -from api.api.schemas import response_schema from api.db.models.base import Base +from api.api.schemas import response_schema + @dataclasses.dataclass class ValidationErrorDetail: diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py index 515bd4b4..21dafa20 100644 --- a/app/api/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -2,14 +2,14 @@ import os.path as path from typing import Optional -import click - import api.adapters.db as db import api.adapters.db.flask_db as flask_db import api.services.users as user_service -from api.api.users.user_blueprint import user_blueprint +import click from api.util.datetime_util import utcnow +from api.api.users.user_blueprint import user_blueprint + logger = logging.getLogger(__name__) user_blueprint.cli.help = "User commands" diff --git a/app/api/api/users/user_routes.py b/app/api/api/users/user_routes.py index 5ef77eee..be1bee6f 100644 --- a/app/api/api/users/user_routes.py +++ b/app/api/api/users/user_routes.py @@ -3,14 +3,15 @@ import api.adapters.db as db import api.adapters.db.flask_db as flask_db -import api.api.response as response -import api.api.users.user_schemas as user_schemas import api.services.users as user_service import api.services.users as users -from api.api.users.user_blueprint import user_blueprint from api.auth.api_key_auth import api_key_auth from api.db.models.user_models import User +import api.api.response as response +import api.api.users.user_schemas as user_schemas +from api.api.users.user_blueprint import user_blueprint + logger = logging.getLogger(__name__) diff --git a/app/api/api/users/user_schemas.py b/app/api/api/users/user_schemas.py index cc44c6b1..03d43700 100644 --- a/app/api/api/users/user_schemas.py +++ b/app/api/api/users/user_schemas.py @@ -1,8 +1,8 @@ +from api.db.models import user_models from apiflask import fields from marshmallow import fields as marshmallow_fields from api.api.schemas import request_schema -from api.db.models import user_models class RoleSchema(request_schema.OrderedSchema): diff --git a/app/api/app.py b/app/api/app.py index 4c118c90..a960ee10 100644 --- a/app/api/app.py +++ b/app/api/app.py @@ -2,18 +2,18 @@ import os from typing import Optional -from apiflask import APIFlask -from flask import g -from werkzeug.exceptions import Unauthorized - import api.adapters.db as db import api.adapters.db.flask_db as flask_db import api.logging import api.logging.flask_logger as flask_logger +from api.auth.api_key_auth import User, get_app_security_scheme +from apiflask import APIFlask +from flask import g +from werkzeug.exceptions import Unauthorized + from api.api.healthcheck import healthcheck_blueprint from api.api.schemas import response_schema from api.api.users import user_blueprint -from api.auth.api_key_auth import User, get_app_security_scheme logger = logging.getLogger(__name__) diff --git a/app/api/db/models/base.py b/app/api/db/models/base.py index 111f688e..170d3442 100644 --- a/app/api/db/models/base.py +++ b/app/api/db/models/base.py @@ -4,14 +4,13 @@ from typing import Any from uuid import UUID +from api.util import datetime_util from sqlalchemy import TIMESTAMP, Column, MetaData, inspect from sqlalchemy.dialects import postgresql from sqlalchemy.ext.declarative import as_declarative from sqlalchemy.orm import declarative_mixin from sqlalchemy.sql.functions import now as sqlnow -from api.util import datetime_util - # Override the default naming of constraints # to use suffixes instead: # https://stackoverflow.com/questions/4107915/postgresql-default-constraint-names/4108266#4108266 diff --git a/app/api/db/models/user_models.py b/app/api/db/models/user_models.py index 6fca730e..e74d9dfd 100644 --- a/app/api/db/models/user_models.py +++ b/app/api/db/models/user_models.py @@ -4,12 +4,11 @@ from typing import Optional from uuid import UUID +from api.db.models.base import Base, IdMixin, TimestampMixin from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Mapped, relationship -from api.db.models.base import Base, IdMixin, TimestampMixin - logger = logging.getLogger(__name__) diff --git a/app/api/services/users/create_user_csv.py b/app/api/services/users/create_user_csv.py index 761d288e..e2465cfe 100644 --- a/app/api/services/users/create_user_csv.py +++ b/app/api/services/users/create_user_csv.py @@ -2,10 +2,9 @@ import logging from dataclasses import asdict, dataclass -from smart_open import open as smart_open - import api.adapters.db as db from api.db.models.user_models import User +from smart_open import open as smart_open logger = logging.getLogger(__name__) diff --git a/app/api/services/users/get_user.py b/app/api/services/users/get_user.py index 2947ad69..f1511ad2 100644 --- a/app/api/services/users/get_user.py +++ b/app/api/services/users/get_user.py @@ -1,8 +1,7 @@ import apiflask -from sqlalchemy import orm - from api.adapters.db import Session from api.db.models.user_models import User +from sqlalchemy import orm # TODO: separate controller and service concerns diff --git a/app/api/services/users/patch_user.py b/app/api/services/users/patch_user.py index c71653e4..680c77fd 100644 --- a/app/api/services/users/patch_user.py +++ b/app/api/services/users/patch_user.py @@ -4,11 +4,10 @@ from typing import TypedDict import apiflask -from sqlalchemy import orm - from api.adapters.db import Session from api.db.models.user_models import Role, User from api.services.users.create_user import RoleParams +from sqlalchemy import orm class PatchUserParams(TypedDict, total=False): diff --git a/app/tests/api/adapters/db/test_db.py b/app/tests/api/adapters/db/test_db.py index de68e044..ca7c1c3a 100644 --- a/app/tests/api/adapters/db/test_db.py +++ b/app/tests/api/adapters/db/test_db.py @@ -1,12 +1,11 @@ import logging # noqa: B1 from itertools import product -import pytest -from sqlalchemy import text - import api.adapters.db as db +import pytest from api.adapters.db.client import get_connection_parameters, make_connection_uri, verify_ssl from api.adapters.db.config import DbConfig, get_db_config +from sqlalchemy import text class DummyConnectionInfo: diff --git a/app/tests/api/adapters/db/test_flask_db.py b/app/tests/api/adapters/db/test_flask_db.py index dfca9964..1943bac8 100644 --- a/app/tests/api/adapters/db/test_flask_db.py +++ b/app/tests/api/adapters/db/test_flask_db.py @@ -1,10 +1,9 @@ +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db import pytest from flask import Flask, current_app from sqlalchemy import text -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db - # Define an isolated example Flask app fixture specific to this test module # to avoid dependencies on any project-specific fixtures in conftest.py diff --git a/app/tests/api/auth/test_api_key_auth.py b/app/tests/api/auth/test_api_key_auth.py index c3434d7b..7d1a853a 100644 --- a/app/tests/api/auth/test_api_key_auth.py +++ b/app/tests/api/auth/test_api_key_auth.py @@ -1,9 +1,8 @@ import pytest +from api.auth.api_key_auth import API_AUTH_USER, verify_token from apiflask import HTTPError from flask import g -from api.auth.api_key_auth import API_AUTH_USER, verify_token - def test_verify_token_success(app, api_auth_token): # Passing it the configured auth token successfully returns a user diff --git a/app/tests/api/db/models/factories.py b/app/tests/api/db/models/factories.py index c822f680..8e6e64fd 100644 --- a/app/tests/api/db/models/factories.py +++ b/app/tests/api/db/models/factories.py @@ -10,22 +10,21 @@ from datetime import datetime from typing import Optional +import api.adapters.db as db +import api.db.models.user_models as user_models +import api.util.datetime_util as datetime_util import factory import factory.fuzzy import faker from sqlalchemy.orm import scoped_session -import api.adapters.db as db -import api.db.models.user_models as user_models -import api.util.datetime_util as datetime_util - _db_session: Optional[db.Session] = None fake = faker.Faker() def get_db_session() -> db.Session: - # _db_session is only set in the pytest fixture `factories_db_session` + # _db_session is only set in the pytest fixture `enable_factory_create` # so that tests do not unintentionally write to the database. if _db_session is None: raise Exception( @@ -36,7 +35,7 @@ def get_db_session() -> db.Session: not persist the generated model. If running tests that actually need data in the DB, pull in the - `factories_db_session` fixture to initialize the db_session. + `enable_factory_create` fixture to initialize the db_session. """ ) diff --git a/app/tests/api/db/models/test_factories.py b/app/tests/api/db/models/test_factories.py index 5b39432c..ab8e0395 100644 --- a/app/tests/api/db/models/test_factories.py +++ b/app/tests/api/db/models/test_factories.py @@ -70,25 +70,25 @@ def test_user_factory_build(): def test_factory_create_uninitialized_db_session(): # DB factory access is disabled from tests unless you add the - # 'factories_db_session' fixture. + # 'enable_factory_create' fixture. with pytest.raises(Exception, match="Factory db_session is not initialized."): UserFactory.create() -def test_user_factory_create(factories_db_session): +def test_user_factory_create(enable_factory_create, db_session: db.Session): # Create actually writes a record to the DB when run # so we'll check the DB directly as well. user = UserFactory.create() validate_user_record(user) - db_record = factories_db_session.query(User).filter(User.id == user.id).one_or_none() + db_record = db_session.query(User).filter(User.id == user.id).one_or_none() # Make certain the DB record matches the factory one. validate_user_record(db_record, user.for_json()) user = UserFactory.create(**user_params) validate_user_record(user, user_params) - db_record = factories_db_session.query(User).filter(User.id == user.id).one_or_none() + db_record = db_session.query(User).filter(User.id == user.id).one_or_none() # Make certain the DB record matches the factory one. validate_user_record(db_record, db_record.for_json()) @@ -97,11 +97,11 @@ def test_user_factory_create(factories_db_session): user = UserFactory.create(**null_params) validate_user_record(user, null_params) - all_db_records = factories_db_session.query(User).all() + all_db_records = db_session.query(User).all() assert len(all_db_records) == 3 -def test_role_factory_create(factories_db_session): +def test_role_factory_create(enable_factory_create): # Verify if you build a Role directly, it gets # a user attached to it with that single role role = RoleFactory.create() diff --git a/app/tests/api/db/test_migrations.py b/app/tests/api/db/test_migrations.py index 0f286631..e5ccd30d 100644 --- a/app/tests/api/db/test_migrations.py +++ b/app/tests/api/db/test_migrations.py @@ -5,7 +5,6 @@ from alembic.script import ScriptDirectory from alembic.script.revision import MultipleHeads from alembic.util.exc import CommandError - from api.db.migrations.run import alembic_cfg diff --git a/app/tests/api/logging/test_audit.py b/app/tests/api/logging/test_audit.py index 874c9a9b..25e58df4 100644 --- a/app/tests/api/logging/test_audit.py +++ b/app/tests/api/logging/test_audit.py @@ -14,9 +14,8 @@ import urllib.request from typing import Any, Callable -import pytest - import api.logging.audit as audit +import pytest # Do not run these tests alongside the rest of the test suite since # this tests adds an audit hook that interfere with other tests, diff --git a/app/tests/api/logging/test_flask_logger.py b/app/tests/api/logging/test_flask_logger.py index c26250b8..4027ae9a 100644 --- a/app/tests/api/logging/test_flask_logger.py +++ b/app/tests/api/logging/test_flask_logger.py @@ -1,10 +1,10 @@ import logging import sys +import api.logging.flask_logger as flask_logger import pytest from flask import Flask -import api.logging.flask_logger as flask_logger from tests.lib.assertions import assert_dict_contains diff --git a/app/tests/api/logging/test_formatters.py b/app/tests/api/logging/test_formatters.py index ecebe241..f9dbbdc6 100644 --- a/app/tests/api/logging/test_formatters.py +++ b/app/tests/api/logging/test_formatters.py @@ -2,9 +2,9 @@ import logging import re +import api.logging.formatters as formatters import pytest -import api.logging.formatters as formatters from tests.lib.assertions import assert_dict_contains diff --git a/app/tests/api/logging/test_logging.py b/app/tests/api/logging/test_logging.py index 53d5a078..ee0347ab 100644 --- a/app/tests/api/logging/test_logging.py +++ b/app/tests/api/logging/test_logging.py @@ -1,10 +1,9 @@ import logging import re -import pytest - import api.logging import api.logging.formatters as formatters +import pytest def _init_test_logger( diff --git a/app/tests/api/logging/test_pii.py b/app/tests/api/logging/test_pii.py index 67c4eedf..041cd4ee 100644 --- a/app/tests/api/logging/test_pii.py +++ b/app/tests/api/logging/test_pii.py @@ -1,6 +1,5 @@ -import pytest - import api.logging.pii as pii +import pytest @pytest.mark.parametrize( diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 319b0135..8743f838 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -20,7 +20,7 @@ def truncate_user(db_session: db.Session): @pytest.fixture -def prepopulated_users(factories_db_session) -> list[User]: +def prepopulated_users(enable_factory_create) -> list[User]: return [ UserFactory.create(first_name="Jon", last_name="Doe", is_active=True), UserFactory.create(first_name="Jane", last_name="Doe", is_active=False), diff --git a/app/tests/api/util/test_datetime_util.py b/app/tests/api/util/test_datetime_util.py index 9a86200c..da80a3da 100644 --- a/app/tests/api/util/test_datetime_util.py +++ b/app/tests/api/util/test_datetime_util.py @@ -2,7 +2,6 @@ import pytest import pytz - from api.util.datetime_util import adjust_timezone diff --git a/app/tests/conftest.py b/app/tests/conftest.py index a1070c93..abcc999c 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,17 +1,17 @@ import logging import _pytest.monkeypatch +import api.adapters.db as db +import api.app as app_entry import boto3 import flask import flask.testing import moto import pytest - -import api.adapters.db as db -import api.app as app_entry -import tests.api.db.models.factories as factories from api.db import models from api.util.local import load_local_env_vars + +import tests.api.db.models.factories as factories from tests.lib import db_testing logger = logging.getLogger(__name__) @@ -72,7 +72,7 @@ def db_session(db_client: db.DBClient) -> db.Session: @pytest.fixture -def factories_db_session(monkeypatch, db_session) -> db.Session: +def enable_factory_create(monkeypatch, db_session) -> db.Session: monkeypatch.setattr(factories, "_db_session", db_session) logger.info("set factories db_session to %s", db_session) return db_session diff --git a/docs/app/database/database-testing.md b/docs/app/database/database-testing.md index 1ccf9ff4..7404bc15 100644 --- a/docs/app/database/database-testing.md +++ b/docs/app/database/database-testing.md @@ -10,4 +10,4 @@ Note that [PostgreSQL schemas](https://www.postgresql.org/docs/current/ddl-schem ## Test Factories -The application uses [Factory Boy](https://factoryboy.readthedocs.io/en/stable/) to generate test data for the application. This can be used to create models `Factory.build` that can be used in any test, or to prepopulate the database with persisted models using `Factory.create`. In order to use `Factory.create` to create persisted database models, include the `factories_db_session` fixture in the test. +The application uses [Factory Boy](https://factoryboy.readthedocs.io/en/stable/) to generate test data for the application. This can be used to create models `Factory.build` that can be used in any test, or to prepopulate the database with persisted models using `Factory.create`. In order to use `Factory.create` to create persisted database models, include the `enable_factory_create` fixture in the test. From 6fc18d13f4930f19882fa1bb2356aab72a68f959 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:22:36 -0800 Subject: [PATCH 42/51] Lint --- app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py | 1 - app/tests/api/scripts/test_create_user_csv.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py b/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py index c793d595..fa443cf4 100644 --- a/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py +++ b/app/api/db/migrations/versions/2023_02_21_cascade_on_delete.py @@ -5,7 +5,6 @@ Create Date: 2023-02-21 18:16:56.052679 """ -import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 8743f838..39068965 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -9,9 +9,7 @@ from pytest_lazyfixture import lazy_fixture from smart_open import open as smart_open -import tests.api.db.models.factories as factories from tests.api.db.models.factories import UserFactory -from tests.lib import db_testing @pytest.fixture From 7e3cfc8c5046207bc8be9b31508a9efd42b8dcec Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:24:42 -0800 Subject: [PATCH 43/51] Rename preopopulate fixture --- app/tests/api/scripts/test_create_user_csv.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 39068965..bffbcdea 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -13,12 +13,12 @@ @pytest.fixture -def truncate_user(db_session: db.Session): +def truncate_user_table(db_session: db.Session): db_session.query(User).delete() @pytest.fixture -def prepopulated_users(enable_factory_create) -> list[User]: +def prepopulate_user_table(enable_factory_create) -> list[User]: return [ UserFactory.create(first_name="Jon", last_name="Doe", is_active=True), UserFactory.create(first_name="Jane", last_name="Doe", is_active=False), @@ -43,9 +43,9 @@ def tmp_s3_folder(mock_s3_bucket): ], ) def test_create_user_csv( - truncate_user, + truncate_user_table, + prepopulate_user_table: list[User], cli_runner: flask.testing.FlaskCliRunner, - prepopulated_users: list[User], dir: str, ): cli_runner.invoke(args=["user", "create-csv", "--dir", dir, "--filename", "test.csv"]) From 1a115af84944723fb87d3549b6999256bf2498e9 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:26:48 -0800 Subject: [PATCH 44/51] Move empty_schema fixture to correct file --- app/tests/api/db/models/test_factories.py | 15 --------------- app/tests/api/db/test_migrations.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/app/tests/api/db/models/test_factories.py b/app/tests/api/db/models/test_factories.py index ab8e0395..064eb801 100644 --- a/app/tests/api/db/models/test_factories.py +++ b/app/tests/api/db/models/test_factories.py @@ -5,21 +5,6 @@ from api.db.models.user_models import User from tests.api.db.models.factories import RoleFactory, UserFactory -from tests.lib import db_testing - - -@pytest.fixture -def empty_schema(monkeypatch) -> db.DBClient: - """ - Create a test schema, if it doesn't already exist, and drop it after the - test completes. - - This is similar to the db fixture but does not create any tables in the - schema. This is used by migration tests. - """ - with db_testing.create_isolated_db(monkeypatch) as db: - yield db - user_params = { "first_name": "Alvin", diff --git a/app/tests/api/db/test_migrations.py b/app/tests/api/db/test_migrations.py index e5ccd30d..005c8840 100644 --- a/app/tests/api/db/test_migrations.py +++ b/app/tests/api/db/test_migrations.py @@ -1,5 +1,6 @@ import logging # noqa: B1 +import api.adapters.db as db import pytest from alembic import command from alembic.script import ScriptDirectory @@ -7,6 +8,21 @@ from alembic.util.exc import CommandError from api.db.migrations.run import alembic_cfg +from tests.lib import db_testing + + +@pytest.fixture +def empty_schema(monkeypatch) -> db.DBClient: + """ + Create a test schema, if it doesn't already exist, and drop it after the + test completes. + + This is similar to the db fixture but does not create any tables in the + schema. This is used by migration tests. + """ + with db_testing.create_isolated_db(monkeypatch) as db: + yield db + def test_only_single_head_revision_in_migrations(): script = ScriptDirectory.from_config(alembic_cfg) From 00be3be7fcfd07b5fc52013b3c22a56690d19435 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:27:05 -0800 Subject: [PATCH 45/51] Combine truncate and prepopulate fixture --- app/tests/api/scripts/test_create_user_csv.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index bffbcdea..bc3b232a 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -13,12 +13,8 @@ @pytest.fixture -def truncate_user_table(db_session: db.Session): +def prepopulate_user_table(enable_factory_create, db_session: db.Session) -> list[User]: db_session.query(User).delete() - - -@pytest.fixture -def prepopulate_user_table(enable_factory_create) -> list[User]: return [ UserFactory.create(first_name="Jon", last_name="Doe", is_active=True), UserFactory.create(first_name="Jane", last_name="Doe", is_active=False), @@ -43,7 +39,6 @@ def tmp_s3_folder(mock_s3_bucket): ], ) def test_create_user_csv( - truncate_user_table, prepopulate_user_table: list[User], cli_runner: flask.testing.FlaskCliRunner, dir: str, From 2c01c76686ca4587c082974cc1d9e093a26231bf Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:32:29 -0800 Subject: [PATCH 46/51] Add comments --- app/tests/api/db/test_migrations.py | 4 ++-- app/tests/api/scripts/test_create_user_csv.py | 1 + app/tests/conftest.py | 13 +++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/app/tests/api/db/test_migrations.py b/app/tests/api/db/test_migrations.py index 005c8840..7f20e33c 100644 --- a/app/tests/api/db/test_migrations.py +++ b/app/tests/api/db/test_migrations.py @@ -17,8 +17,8 @@ def empty_schema(monkeypatch) -> db.DBClient: Create a test schema, if it doesn't already exist, and drop it after the test completes. - This is similar to the db fixture but does not create any tables in the - schema. This is used by migration tests. + This is similar to what the db_client fixture does but does not create any tables in the + schema. """ with db_testing.create_isolated_db(monkeypatch) as db: yield db diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index bc3b232a..90fd39ab 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -14,6 +14,7 @@ @pytest.fixture def prepopulate_user_table(enable_factory_create, db_session: db.Session) -> list[User]: + # First make sure the table is empty db_session.query(User).delete() return [ UserFactory.create(first_name="Jon", last_name="Doe", is_active=True), diff --git a/app/tests/conftest.py b/app/tests/conftest.py index abcc999c..233ed400 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -54,7 +54,7 @@ def db_client(monkeypatch_session) -> db.DBClient: Creates an isolated database for the test session. Creates a new empty PostgreSQL schema, creates all tables in the new schema - using SQLAlchemy, then returns a db.DB instance that can be used to + using SQLAlchemy, then returns a db.DBClient instance that can be used to get connections or sessions to this database schema. The schema is dropped after the test suite session completes. """ @@ -66,13 +66,22 @@ def db_client(monkeypatch_session) -> db.DBClient: @pytest.fixture def db_session(db_client: db.DBClient) -> db.Session: - # Based on https://docs.sqlalchemy.org/en/13/orm/session_transaction.html#joining-a-session-into-an-external-transaction-such-as-for-test-suites + """ + Returns a database session connected to the schema used for the test session. + """ with db_client.get_session() as session: yield session @pytest.fixture def enable_factory_create(monkeypatch, db_session) -> db.Session: + """ + Allows the create method of factories to be called. By default, the create + throws an exception to prevent accidental creation of database objects for tests + that do not need persistence. This fixture only allows the create method to be + called for the current test. Each test that needs to call Factory.create should pull in + this fixture. + """ monkeypatch.setattr(factories, "_db_session", db_session) logger.info("set factories db_session to %s", db_session) return db_session From 83ccce0f0ca7fef18a8ac919e8e447190d69a05e Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 10:53:52 -0800 Subject: [PATCH 47/51] Rename db var to avoid name conflict --- app/tests/api/db/test_migrations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/api/db/test_migrations.py b/app/tests/api/db/test_migrations.py index 7f20e33c..3553ed42 100644 --- a/app/tests/api/db/test_migrations.py +++ b/app/tests/api/db/test_migrations.py @@ -20,8 +20,8 @@ def empty_schema(monkeypatch) -> db.DBClient: This is similar to what the db_client fixture does but does not create any tables in the schema. """ - with db_testing.create_isolated_db(monkeypatch) as db: - yield db + with db_testing.create_isolated_db(monkeypatch) as db_client: + yield db_client def test_only_single_head_revision_in_migrations(): From 81868e9c86e8cf21249bb258a236de67db1573b8 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 11:51:26 -0800 Subject: [PATCH 48/51] isort --- app/api/adapters/db/client.py | 3 ++- app/api/adapters/db/config.py | 3 ++- app/api/adapters/db/flask_db.py | 3 ++- app/api/api/healthcheck.py | 2 +- app/api/api/response.py | 3 +-- app/api/api/users/user_commands.py | 6 +++--- app/api/api/users/user_routes.py | 7 +++---- app/api/api/users/user_schemas.py | 2 +- app/api/app.py | 10 +++++----- app/api/db/models/base.py | 3 ++- app/api/db/models/user_models.py | 3 ++- app/api/services/users/create_user_csv.py | 3 ++- app/api/services/users/get_user.py | 3 ++- app/api/services/users/patch_user.py | 3 ++- app/tests/api/adapters/db/test_db.py | 5 +++-- app/tests/api/adapters/db/test_flask_db.py | 5 +++-- app/tests/api/auth/test_api_key_auth.py | 3 ++- app/tests/api/db/models/factories.py | 7 ++++--- app/tests/api/db/models/test_factories.py | 4 ++-- app/tests/api/db/test_migrations.py | 4 ++-- app/tests/api/logging/test_audit.py | 3 ++- app/tests/api/logging/test_flask_logger.py | 2 +- app/tests/api/logging/test_formatters.py | 2 +- app/tests/api/logging/test_logging.py | 3 ++- app/tests/api/logging/test_pii.py | 3 ++- app/tests/api/scripts/test_create_user_csv.py | 4 ++-- app/tests/api/util/test_datetime_util.py | 1 + app/tests/conftest.py | 8 ++++---- 28 files changed, 61 insertions(+), 47 deletions(-) diff --git a/app/api/adapters/db/client.py b/app/api/adapters/db/client.py index 585bbdc2..36f38eee 100644 --- a/app/api/adapters/db/client.py +++ b/app/api/adapters/db/client.py @@ -15,9 +15,10 @@ import psycopg2 import sqlalchemy import sqlalchemy.pool as pool -from api.adapters.db.config import DbConfig, get_db_config from sqlalchemy.orm import session +from api.adapters.db.config import DbConfig, get_db_config + # Re-export the Connection type that is returned by the get_connection() method # to be used for type hints. Connection = sqlalchemy.engine.Connection diff --git a/app/api/adapters/db/config.py b/app/api/adapters/db/config.py index a7edc9dd..67475f65 100644 --- a/app/api/adapters/db/config.py +++ b/app/api/adapters/db/config.py @@ -1,9 +1,10 @@ import logging from typing import Optional -from api.util.env_config import PydanticBaseEnvConfig from pydantic import Field +from api.util.env_config import PydanticBaseEnvConfig + logger = logging.getLogger(__name__) diff --git a/app/api/adapters/db/flask_db.py b/app/api/adapters/db/flask_db.py index 788c2e73..04f486c3 100644 --- a/app/api/adapters/db/flask_db.py +++ b/app/api/adapters/db/flask_db.py @@ -41,9 +41,10 @@ def health(): from functools import wraps from typing import Any, Callable, Concatenate, ParamSpec, TypeVar +from flask import Flask, current_app + import api.adapters.db as db from api.adapters.db.client import DBClient -from flask import Flask, current_app _FLASK_EXTENSION_KEY = "db" diff --git a/app/api/api/healthcheck.py b/app/api/api/healthcheck.py index c3d80f04..4df442b2 100644 --- a/app/api/api/healthcheck.py +++ b/app/api/api/healthcheck.py @@ -1,12 +1,12 @@ import logging from typing import Tuple -import api.adapters.db.flask_db as flask_db from apiflask import APIBlueprint from flask import current_app from sqlalchemy import text from werkzeug.exceptions import ServiceUnavailable +import api.adapters.db.flask_db as flask_db from api.api import response from api.api.schemas import request_schema diff --git a/app/api/api/response.py b/app/api/api/response.py index 86da4643..65f6e0bb 100644 --- a/app/api/api/response.py +++ b/app/api/api/response.py @@ -1,9 +1,8 @@ import dataclasses from typing import Optional -from api.db.models.base import Base - from api.api.schemas import response_schema +from api.db.models.base import Base @dataclasses.dataclass diff --git a/app/api/api/users/user_commands.py b/app/api/api/users/user_commands.py index 21dafa20..515bd4b4 100644 --- a/app/api/api/users/user_commands.py +++ b/app/api/api/users/user_commands.py @@ -2,13 +2,13 @@ import os.path as path from typing import Optional +import click + import api.adapters.db as db import api.adapters.db.flask_db as flask_db import api.services.users as user_service -import click -from api.util.datetime_util import utcnow - from api.api.users.user_blueprint import user_blueprint +from api.util.datetime_util import utcnow logger = logging.getLogger(__name__) diff --git a/app/api/api/users/user_routes.py b/app/api/api/users/user_routes.py index be1bee6f..5ef77eee 100644 --- a/app/api/api/users/user_routes.py +++ b/app/api/api/users/user_routes.py @@ -3,15 +3,14 @@ import api.adapters.db as db import api.adapters.db.flask_db as flask_db +import api.api.response as response +import api.api.users.user_schemas as user_schemas import api.services.users as user_service import api.services.users as users +from api.api.users.user_blueprint import user_blueprint from api.auth.api_key_auth import api_key_auth from api.db.models.user_models import User -import api.api.response as response -import api.api.users.user_schemas as user_schemas -from api.api.users.user_blueprint import user_blueprint - logger = logging.getLogger(__name__) diff --git a/app/api/api/users/user_schemas.py b/app/api/api/users/user_schemas.py index 03d43700..cc44c6b1 100644 --- a/app/api/api/users/user_schemas.py +++ b/app/api/api/users/user_schemas.py @@ -1,8 +1,8 @@ -from api.db.models import user_models from apiflask import fields from marshmallow import fields as marshmallow_fields from api.api.schemas import request_schema +from api.db.models import user_models class RoleSchema(request_schema.OrderedSchema): diff --git a/app/api/app.py b/app/api/app.py index a960ee10..4c118c90 100644 --- a/app/api/app.py +++ b/app/api/app.py @@ -2,18 +2,18 @@ import os from typing import Optional -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db -import api.logging -import api.logging.flask_logger as flask_logger -from api.auth.api_key_auth import User, get_app_security_scheme from apiflask import APIFlask from flask import g from werkzeug.exceptions import Unauthorized +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db +import api.logging +import api.logging.flask_logger as flask_logger from api.api.healthcheck import healthcheck_blueprint from api.api.schemas import response_schema from api.api.users import user_blueprint +from api.auth.api_key_auth import User, get_app_security_scheme logger = logging.getLogger(__name__) diff --git a/app/api/db/models/base.py b/app/api/db/models/base.py index 170d3442..111f688e 100644 --- a/app/api/db/models/base.py +++ b/app/api/db/models/base.py @@ -4,13 +4,14 @@ from typing import Any from uuid import UUID -from api.util import datetime_util from sqlalchemy import TIMESTAMP, Column, MetaData, inspect from sqlalchemy.dialects import postgresql from sqlalchemy.ext.declarative import as_declarative from sqlalchemy.orm import declarative_mixin from sqlalchemy.sql.functions import now as sqlnow +from api.util import datetime_util + # Override the default naming of constraints # to use suffixes instead: # https://stackoverflow.com/questions/4107915/postgresql-default-constraint-names/4108266#4108266 diff --git a/app/api/db/models/user_models.py b/app/api/db/models/user_models.py index e74d9dfd..6fca730e 100644 --- a/app/api/db/models/user_models.py +++ b/app/api/db/models/user_models.py @@ -4,11 +4,12 @@ from typing import Optional from uuid import UUID -from api.db.models.base import Base, IdMixin, TimestampMixin from sqlalchemy import Boolean, Column, Date, Enum, ForeignKey, Text from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Mapped, relationship +from api.db.models.base import Base, IdMixin, TimestampMixin + logger = logging.getLogger(__name__) diff --git a/app/api/services/users/create_user_csv.py b/app/api/services/users/create_user_csv.py index e2465cfe..761d288e 100644 --- a/app/api/services/users/create_user_csv.py +++ b/app/api/services/users/create_user_csv.py @@ -2,9 +2,10 @@ import logging from dataclasses import asdict, dataclass +from smart_open import open as smart_open + import api.adapters.db as db from api.db.models.user_models import User -from smart_open import open as smart_open logger = logging.getLogger(__name__) diff --git a/app/api/services/users/get_user.py b/app/api/services/users/get_user.py index f1511ad2..2947ad69 100644 --- a/app/api/services/users/get_user.py +++ b/app/api/services/users/get_user.py @@ -1,7 +1,8 @@ import apiflask +from sqlalchemy import orm + from api.adapters.db import Session from api.db.models.user_models import User -from sqlalchemy import orm # TODO: separate controller and service concerns diff --git a/app/api/services/users/patch_user.py b/app/api/services/users/patch_user.py index 680c77fd..c71653e4 100644 --- a/app/api/services/users/patch_user.py +++ b/app/api/services/users/patch_user.py @@ -4,10 +4,11 @@ from typing import TypedDict import apiflask +from sqlalchemy import orm + from api.adapters.db import Session from api.db.models.user_models import Role, User from api.services.users.create_user import RoleParams -from sqlalchemy import orm class PatchUserParams(TypedDict, total=False): diff --git a/app/tests/api/adapters/db/test_db.py b/app/tests/api/adapters/db/test_db.py index ca7c1c3a..de68e044 100644 --- a/app/tests/api/adapters/db/test_db.py +++ b/app/tests/api/adapters/db/test_db.py @@ -1,11 +1,12 @@ import logging # noqa: B1 from itertools import product -import api.adapters.db as db import pytest +from sqlalchemy import text + +import api.adapters.db as db from api.adapters.db.client import get_connection_parameters, make_connection_uri, verify_ssl from api.adapters.db.config import DbConfig, get_db_config -from sqlalchemy import text class DummyConnectionInfo: diff --git a/app/tests/api/adapters/db/test_flask_db.py b/app/tests/api/adapters/db/test_flask_db.py index 1943bac8..dfca9964 100644 --- a/app/tests/api/adapters/db/test_flask_db.py +++ b/app/tests/api/adapters/db/test_flask_db.py @@ -1,9 +1,10 @@ -import api.adapters.db as db -import api.adapters.db.flask_db as flask_db import pytest from flask import Flask, current_app from sqlalchemy import text +import api.adapters.db as db +import api.adapters.db.flask_db as flask_db + # Define an isolated example Flask app fixture specific to this test module # to avoid dependencies on any project-specific fixtures in conftest.py diff --git a/app/tests/api/auth/test_api_key_auth.py b/app/tests/api/auth/test_api_key_auth.py index 7d1a853a..c3434d7b 100644 --- a/app/tests/api/auth/test_api_key_auth.py +++ b/app/tests/api/auth/test_api_key_auth.py @@ -1,8 +1,9 @@ import pytest -from api.auth.api_key_auth import API_AUTH_USER, verify_token from apiflask import HTTPError from flask import g +from api.auth.api_key_auth import API_AUTH_USER, verify_token + def test_verify_token_success(app, api_auth_token): # Passing it the configured auth token successfully returns a user diff --git a/app/tests/api/db/models/factories.py b/app/tests/api/db/models/factories.py index 8e6e64fd..811d634d 100644 --- a/app/tests/api/db/models/factories.py +++ b/app/tests/api/db/models/factories.py @@ -10,14 +10,15 @@ from datetime import datetime from typing import Optional -import api.adapters.db as db -import api.db.models.user_models as user_models -import api.util.datetime_util as datetime_util import factory import factory.fuzzy import faker from sqlalchemy.orm import scoped_session +import api.adapters.db as db +import api.db.models.user_models as user_models +import api.util.datetime_util as datetime_util + _db_session: Optional[db.Session] = None fake = faker.Faker() diff --git a/app/tests/api/db/models/test_factories.py b/app/tests/api/db/models/test_factories.py index 064eb801..f1500501 100644 --- a/app/tests/api/db/models/test_factories.py +++ b/app/tests/api/db/models/test_factories.py @@ -1,9 +1,9 @@ from datetime import date, datetime -import api.adapters.db as db import pytest -from api.db.models.user_models import User +import api.adapters.db as db +from api.db.models.user_models import User from tests.api.db.models.factories import RoleFactory, UserFactory user_params = { diff --git a/app/tests/api/db/test_migrations.py b/app/tests/api/db/test_migrations.py index 3553ed42..60a0babd 100644 --- a/app/tests/api/db/test_migrations.py +++ b/app/tests/api/db/test_migrations.py @@ -1,13 +1,13 @@ import logging # noqa: B1 -import api.adapters.db as db import pytest from alembic import command from alembic.script import ScriptDirectory from alembic.script.revision import MultipleHeads from alembic.util.exc import CommandError -from api.db.migrations.run import alembic_cfg +import api.adapters.db as db +from api.db.migrations.run import alembic_cfg from tests.lib import db_testing diff --git a/app/tests/api/logging/test_audit.py b/app/tests/api/logging/test_audit.py index 9f06a9c4..80d09969 100644 --- a/app/tests/api/logging/test_audit.py +++ b/app/tests/api/logging/test_audit.py @@ -14,9 +14,10 @@ import urllib.request from typing import Any, Callable -import api.logging.audit as audit import pytest +import api.logging.audit as audit + # Do not run these tests alongside the rest of the test suite since # this tests adds an audit hook that interfere with other tests, # and at the time of writing there isn't a known way to remove diff --git a/app/tests/api/logging/test_flask_logger.py b/app/tests/api/logging/test_flask_logger.py index 3d49a121..b2bb14c3 100644 --- a/app/tests/api/logging/test_flask_logger.py +++ b/app/tests/api/logging/test_flask_logger.py @@ -2,10 +2,10 @@ import sys import time -import api.logging.flask_logger as flask_logger import pytest from flask import Flask +import api.logging.flask_logger as flask_logger from tests.lib.assertions import assert_dict_contains diff --git a/app/tests/api/logging/test_formatters.py b/app/tests/api/logging/test_formatters.py index f9dbbdc6..ecebe241 100644 --- a/app/tests/api/logging/test_formatters.py +++ b/app/tests/api/logging/test_formatters.py @@ -2,9 +2,9 @@ import logging import re -import api.logging.formatters as formatters import pytest +import api.logging.formatters as formatters from tests.lib.assertions import assert_dict_contains diff --git a/app/tests/api/logging/test_logging.py b/app/tests/api/logging/test_logging.py index ee0347ab..53d5a078 100644 --- a/app/tests/api/logging/test_logging.py +++ b/app/tests/api/logging/test_logging.py @@ -1,9 +1,10 @@ import logging import re +import pytest + import api.logging import api.logging.formatters as formatters -import pytest def _init_test_logger( diff --git a/app/tests/api/logging/test_pii.py b/app/tests/api/logging/test_pii.py index 041cd4ee..67c4eedf 100644 --- a/app/tests/api/logging/test_pii.py +++ b/app/tests/api/logging/test_pii.py @@ -1,6 +1,7 @@ -import api.logging.pii as pii import pytest +import api.logging.pii as pii + @pytest.mark.parametrize( "input,expected", diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index 90fd39ab..d001a4d8 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -2,13 +2,13 @@ import os.path as path import re -import api.adapters.db as db import flask.testing import pytest -from api.db.models.user_models import User from pytest_lazyfixture import lazy_fixture from smart_open import open as smart_open +import api.adapters.db as db +from api.db.models.user_models import User from tests.api.db.models.factories import UserFactory diff --git a/app/tests/api/util/test_datetime_util.py b/app/tests/api/util/test_datetime_util.py index da80a3da..9a86200c 100644 --- a/app/tests/api/util/test_datetime_util.py +++ b/app/tests/api/util/test_datetime_util.py @@ -2,6 +2,7 @@ import pytest import pytz + from api.util.datetime_util import adjust_timezone diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 233ed400..842251b0 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,17 +1,17 @@ import logging import _pytest.monkeypatch -import api.adapters.db as db -import api.app as app_entry import boto3 import flask import flask.testing import moto import pytest -from api.db import models -from api.util.local import load_local_env_vars +import api.adapters.db as db +import api.app as app_entry import tests.api.db.models.factories as factories +from api.db import models +from api.util.local import load_local_env_vars from tests.lib import db_testing logger = logging.getLogger(__name__) From 873939c16607ad778c4e39978ce89e278604e4ce Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 12:00:10 -0800 Subject: [PATCH 49/51] Add known first party --- app/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/pyproject.toml b/app/pyproject.toml index eb85e2cd..7925d63e 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -57,6 +57,7 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true line_length = 100 +known_first_party = ["api"] [tool.mypy] # https://mypy.readthedocs.io/en/stable/config_file.html From 1e2788b7c7dea3d13728ca7c8094d3c6117bf15a Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 12:55:09 -0800 Subject: [PATCH 50/51] Revert "Add known first party" This reverts commit 873939c16607ad778c4e39978ce89e278604e4ce. --- app/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/app/pyproject.toml b/app/pyproject.toml index 7925d63e..eb85e2cd 100644 --- a/app/pyproject.toml +++ b/app/pyproject.toml @@ -57,7 +57,6 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true line_length = 100 -known_first_party = ["api"] [tool.mypy] # https://mypy.readthedocs.io/en/stable/config_file.html From 0a837a80f5cd149810a3b61abb494f8fccd24e72 Mon Sep 17 00:00:00 2001 From: Loren Yu Date: Tue, 21 Feb 2023 14:42:23 -0800 Subject: [PATCH 51/51] Add comment --- app/tests/api/scripts/test_create_user_csv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/tests/api/scripts/test_create_user_csv.py b/app/tests/api/scripts/test_create_user_csv.py index d001a4d8..31f0c577 100644 --- a/app/tests/api/scripts/test_create_user_csv.py +++ b/app/tests/api/scripts/test_create_user_csv.py @@ -14,7 +14,9 @@ @pytest.fixture def prepopulate_user_table(enable_factory_create, db_session: db.Session) -> list[User]: - # First make sure the table is empty + # First make sure the table is empty, as other tests may have inserted data + # and this test expects a clean slate (unlike most tests that are designed to + # be isolated from other tests) db_session.query(User).delete() return [ UserFactory.create(first_name="Jon", last_name="Doe", is_active=True),