Experiment: multi-environment configuration in Python

app/Dockerfile

@@ -28,7 +28,7 @@ COPY . /srv
 
 # Set the host to 0.0.0.0 to make the server available external
 # to the Docker container that it's running in.
-ENV HOST=0.0.0.0
+ENV APP_HOST=0.0.0.0
 
 # Install application dependencies.
 # https://python-poetry.org/docs/basic-usage/#installing-dependencies

app/Makefile

@@ -16,9 +16,9 @@ DECODE_LOG := 2>&1 | python3 -u src/logging/util/decodelog.py
 # TODO - when CI gets hooked up, actually test this.
 ifdef CI
  DOCKER_EXEC_ARGS := -T -e CI -e PYTEST_ADDOPTS="--color=yes"
- FLAKE8_FORMAT := '::warning file=src/%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s'
+ FLAKE8_FORMAT := '::warning file=app/%(path)s,line=%(row)d,col=%(col)d::%(path)s:%(row)d:%(col)d: %(code)s %(text)s'
  MYPY_FLAGS := --no-pretty
- MYPY_POSTPROC := | perl -pe "s/^(.+):(\d+):(\d+): error: (.*)/::warning file=src\/\1,line=\2,col=\3::\4/"
+ MYPY_POSTPROC := | perl -pe "s/^(.+):(\d+):(\d+): error: (.*)/::warning file=app\/\1,line=\2,col=\3::\4/"
 else
  FLAKE8_FORMAT := default
 endif
@@ -33,7 +33,7 @@ else
 PY_RUN_CMD := docker-compose run $(DOCKER_EXEC_ARGS) --rm $(APP_NAME) poetry run
 endif
 
-FLASK_CMD := $(PY_RUN_CMD) flask --env-file local.env
+FLASK_CMD := $(PY_RUN_CMD) flask --app=src.__main__:main
 
 ##################################################
 # Local Development Environment Setup

app/src/__main__.py

@@ -6,39 +6,59 @@
 # https://docs.python.org/3/library/__main__.html
 
 import logging
+import os
+
+from flask import Flask
 
 import src.app
+import src.config
+import src.config.load
 import src.logging
-from src.app_config import AppConfig
-from src.util.local import load_local_env_vars
 
 logger = logging.getLogger(__package__)
 
 
-def main() -> None:
-    load_local_env_vars()
-    app_config = AppConfig()
+def load_config() -> src.config.RootConfig:
+    return src.config.load.load(
+        environment_name=os.getenv("ENVIRONMENT", "local"), environ=os.environ
+    )
+
 
-    app = src.app.create_app()
+def main() -> Flask:
+    config = load_config()
+    app = src.app.create_app(config)
+    logger.info("loaded configuration", extra={"config": config})
 
-    environment = app_config.environment
+    environment = config.app.environment
 
     # When running in a container, the host needs to be set to 0.0.0.0 so that the app can be
     # accessed from outside the container. See Dockerfile
-    host = app_config.host
-    port = app_config.port
+    host = config.app.host
+    port = config.app.port
+
+    if __name__ != "__main__":
+        return app
 
     logger.info(
         "Running API Application", extra={"environment": environment, "host": host, "port": port}
     )
 
-    if app_config.environment == "local":
+    if config.app.environment == "local":
         # If python files are changed, the app will auto-reload
         # Note this doesn't have the OpenAPI yaml file configured at the moment
-        app.run(host=host, port=port, use_reloader=True, reloader_type="stat")
+        app.run(
+            host=host,
+            port=port,
+            debug=True,  # nosec B201
+            load_dotenv=False,
+            use_reloader=True,
+            reloader_type="stat",
+        )
     else:
         # Don't enable the reloader if non-local
-        app.run(host=host, port=port)
+        app.run(host=host, port=port, load_dotenv=False)
+
+    return app
 
 
 main()

app/src/adapters/db/clients/postgres_client.py

@@ -1,5 +1,4 @@
 import logging
-import os
 import urllib.parse
 from typing import Any
 
@@ -8,7 +7,7 @@
 import sqlalchemy.pool as pool
 
 from src.adapters.db.client import DBClient
-from src.adapters.db.clients.postgres_config import PostgresDBConfig, get_db_config
+from src.adapters.db.clients.postgres_config import PostgresDBConfig
 
 logger = logging.getLogger(__name__)
 
@@ -19,9 +18,7 @@ class PostgresDBClient(DBClient):
     as configured by parameters passed in from the db_config
     """
 
-    def __init__(self, db_config: PostgresDBConfig | None = None) -> None:
-        if not db_config:
-            db_config = get_db_config()
+    def __init__(self, db_config: PostgresDBConfig) -> None:
         self._engine = self._configure_engine(db_config)
 
         if db_config.check_connection_on_init:
@@ -80,23 +77,15 @@ def check_db_connection(self) -> None:
 
 
 def get_connection_parameters(db_config: PostgresDBConfig) -> dict[str, Any]:
-    connect_args = {}
-    environment = os.getenv("ENVIRONMENT")
-    if not environment:
-        raise Exception("ENVIRONMENT is not set")
-
-    if environment != "local":
-        connect_args["sslmode"] = "require"
-
     return dict(
         host=db_config.host,
         dbname=db_config.name,
         user=db_config.username,
-        password=db_config.password,
+        password=db_config.password.get_secret_value() if db_config.password else None,
         port=db_config.port,
         options=f"-c search_path={db_config.db_schema}",
         connect_timeout=3,
-        **connect_args,
+        sslmode=db_config.sslmode,
     )
 
 
@@ -109,7 +98,7 @@ def make_connection_uri(config: PostgresDBConfig) -> str:
     host = config.host
     db_name = config.name
     username = config.username
-    password = urllib.parse.quote(config.password) if config.password else None
+    password = urllib.parse.quote(config.password.get_secret_value()) if config.password else None
     schema = config.db_schema
     port = config.port
 
@@ -122,10 +111,7 @@ def make_connection_uri(config: PostgresDBConfig) -> str:
     elif password:
         netloc_parts.append(f":{password}@")
 
-    netloc_parts.append(host)
-
-    if port:
-        netloc_parts.append(f":{port}")
+    netloc_parts.append(f"{host}:{port}")
 
     netloc = "".join(netloc_parts)
 

app/src/adapters/db/clients/postgres_config.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Optional
 
+import pydantic
 from pydantic import Field
 
 from src.util.env_config import PydanticBaseEnvConfig
@@ -13,26 +14,8 @@ class PostgresDBConfig(PydanticBaseEnvConfig):
     host: str = Field("localhost", env="DB_HOST")
     name: str = Field("main-db", env="POSTGRES_DB")
     username: str = Field("local_db_user", env="POSTGRES_USER")
-    password: Optional[str] = Field(..., env="POSTGRES_PASSWORD")
+    password: Optional[pydantic.types.SecretStr] = Field(None, env="POSTGRES_PASSWORD")
     db_schema: str = Field("public", env="DB_SCHEMA")
-    port: str = Field("5432", env="DB_PORT")
+    port: int = Field(5432, env="DB_PORT")
     hide_sql_parameter_logs: bool = Field(True, env="HIDE_SQL_PARAMETER_LOGS")
-
-
-def get_db_config() -> PostgresDBConfig:
-    db_config = PostgresDBConfig()
-
-    logger.info(
-        "Constructed database configuration",
-        extra={
-            "host": db_config.host,
-            "dbname": db_config.name,
-            "username": db_config.username,
-            "password": "***" if db_config.password is not None else None,
-            "db_schema": db_config.db_schema,
-            "port": db_config.port,
-            "hide_sql_parameter_logs": db_config.hide_sql_parameter_logs,
-        },
-    )
-
-    return db_config
+    sslmode: str = "require"

app/src/app.py

@@ -14,17 +14,18 @@
 from src.api.schemas import response_schema
 from src.api.users import user_blueprint
 from src.auth.api_key_auth import User, get_app_security_scheme
+from src.config import RootConfig
 
 logger = logging.getLogger(__name__)
 
 
-def create_app() -> APIFlask:
+def create_app(config: RootConfig) -> APIFlask:
     app = APIFlask(__name__)
 
-    src.logging.init(__package__)
+    src.logging.init(__package__, config.logging)
     flask_logger.init_app(logging.root, app)
 
-    db_client = db.PostgresDBClient()
+    db_client = db.PostgresDBClient(config.database)
     flask_db.register_db_client(db_client, app)
 
     configure_app(app)

app/src/app_config.py

@@ -2,7 +2,7 @@
 
 
 class AppConfig(PydanticBaseEnvConfig):
-    environment: str
+    environment: str = "unknown"
 
     # Set HOST to 127.0.0.1 by default to avoid other machines on the network
     # from accessing the application. This is especially important if you are

app/src/config/__init__.py

@@ -0,0 +1,14 @@
+#
+# Multi-environment configuration expressed in Python.
+#
+
+from src.adapters.db.clients.postgres_config import PostgresDBConfig
+from src.app_config import AppConfig
+from src.logging.config import LoggingConfig
+from src.util.env_config import PydanticBaseEnvConfig
+
+
+class RootConfig(PydanticBaseEnvConfig):
+    app: AppConfig
+    database: PostgresDBConfig
+    logging: LoggingConfig

app/src/config/default.py

@@ -0,0 +1,26 @@
+#
+# Default configuration.
+#
+# This is the base layer of configuration. It is used if an environment does not override a value.
+# Each environment may override individual values (see local.py, dev.py, prod.py, etc.).
+#
+# This configuration is also used when running tests (individual test cases may have code to use
+# different configuration).
+#
+
+from src.adapters.db.clients.postgres_config import PostgresDBConfig
+from src.app_config import AppConfig
+from src.config import RootConfig
+from src.logging.config import LoggingConfig, LoggingFormat
+
+
+def default_config() -> RootConfig:
+    return RootConfig(
+        app=AppConfig(),
+        database=PostgresDBConfig(),
+        logging=LoggingConfig(
+            format=LoggingFormat.json,
+            level="INFO",
+            enable_audit=True,
+        ),
+    )

app/src/config/env/dev.py

@@ -0,0 +1,9 @@
+#
+# Configuration for dev environments.
+#
+# This file only contains overrides (differences) from the defaults in default.py.
+#
+
+from .. import default
+
+config = default.default_config()

app/src/config/env/local.py

@@ -0,0 +1,19 @@
+#
+# Configuration for local development environments.
+#
+# This file only contains overrides (differences) from the defaults in default.py.
+#
+
+import pydantic.types
+
+from src.logging.config import LoggingFormat
+
+from .. import default
+
+config = default.default_config()
+
+config.database.password = pydantic.types.SecretStr("secret123")
+config.database.hide_sql_parameter_logs = False
+config.database.sslmode = "prefer"
+config.logging.format = LoggingFormat.human_readable
+config.logging.enable_audit = False