-
Notifications
You must be signed in to change notification settings - Fork 5
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Experiment: multi-environment configuration in Python #161
base: main
Are you sure you want to change the base?
Changes from 2 commits
44eccbf
17f93c8
3d4da44
2da8195
4194db8
d1c90d2
e86e683
1fec89a
e25d471
ca4704e
d62bd78
f7df2b8
e56ab12
4b29c4e
d54a16e
dd7e4d6
2a01d3f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,7 +2,7 @@ | |
|
||
|
||
class AppConfig(PydanticBaseEnvConfig): | ||
environment: str | ||
environment: str = "unknown" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe |
||
|
||
# Set HOST to 127.0.0.1 by default to avoid other machines on the network | ||
# from accessing the application. This is especially important if you are | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
# | ||
# Multi-environment configuration expressed in Python. | ||
# | ||
|
||
import importlib | ||
import logging | ||
import pathlib | ||
|
||
from src.adapters.db.clients.postgres_config import PostgresDBConfig | ||
from src.app_config import AppConfig | ||
from src.logging.config import LoggingConfig | ||
from src.util.env_config import PydanticBaseEnvConfig | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
class RootConfig(PydanticBaseEnvConfig): | ||
app: AppConfig | ||
database: PostgresDBConfig | ||
logging: LoggingConfig | ||
|
||
|
||
def load(environment_name, environ=None) -> RootConfig: | ||
"""Load the configuration for the given environment name.""" | ||
logger.info("loading configuration", extra={"environment": environment_name}) | ||
module = importlib.import_module(name=".env." + environment_name, package=__package__) | ||
config = module.config | ||
|
||
if environ: | ||
config.override_from_environment(environ) | ||
config.app.environment = environment_name | ||
|
||
return config | ||
|
||
|
||
def load_all() -> dict[str, RootConfig]: | ||
"""Load all environment configurations, to ensure they are valid. Used in tests.""" | ||
directory = pathlib.Path(__file__).parent / "env" | ||
return {item.stem: load(str(item.stem)) for item in directory.glob("*.py")} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
# | ||
# Default configuration. | ||
# | ||
# This is the base layer of configuration. It is used if an environment does not override a value. | ||
# Each environment may override individual values (see local.py, dev.py, prod.py, etc.). | ||
# | ||
# This configuration is also used when running tests (individual test cases may have code to use | ||
# different configuration). | ||
# | ||
|
||
from src.adapters.db.clients.postgres_config import PostgresDBConfig | ||
from src.app_config import AppConfig | ||
from src.config import RootConfig | ||
from src.logging.config import LoggingConfig | ||
|
||
|
||
def default_config(): | ||
return RootConfig( | ||
app=AppConfig(), | ||
database=PostgresDBConfig(), | ||
logging=LoggingConfig( | ||
format="json", | ||
level="INFO", | ||
enable_audit=True, | ||
), | ||
) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would this require any process that runs to be able to load all environment variables? For example, if I have a config for connecting to a separate service, and it's only used in a few select processes, would I put that here in the default config? Let's assume that I don't want to set defaults in the config and want it to error if any fields aren't set. |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# | ||
# Configuration for dev environments. | ||
# | ||
# This file only contains overrides (differences) from the defaults in default.py. | ||
# | ||
|
||
from .. import default | ||
|
||
config = default.default_config() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
# | ||
# Configuration for local development environments. | ||
# | ||
# This file only contains overrides (differences) from the defaults in default.py. | ||
# | ||
|
||
import pydantic.types | ||
|
||
from .. import default | ||
|
||
config = default.default_config() | ||
|
||
config.database.password = pydantic.types.SecretStr("secret123") | ||
config.database.hide_sql_parameter_logs = False | ||
config.logging.format = "human_readable" | ||
config.logging.enable_audit = False | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Would there be a way to have this check for a file named Adding something like this to the end of this file: if os.path.exists("local_overrides.py"):
importlib.import_module("local_overrides") # whatever the right path/params are |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# | ||
# Configuration for prod environment. | ||
# | ||
# This file only contains overrides (differences) from the defaults in default.py. | ||
# | ||
|
||
from .. import default | ||
|
||
config = default.default_config() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# | ||
# Configuration for staging environment. | ||
# | ||
# This file only contains overrides (differences) from the defaults in default.py. | ||
# | ||
|
||
from .. import default | ||
|
||
config = default.default_config() |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,30 @@ | ||
import os | ||
import logging | ||
|
||
from pydantic import BaseSettings | ||
from pydantic import BaseModel | ||
|
||
import src | ||
logger = logging.getLogger(__name__) | ||
|
||
env_file = os.path.join( | ||
os.path.dirname(os.path.dirname(src.__file__)), | ||
"config", | ||
"%s.env" % os.getenv("ENVIRONMENT", "local"), | ||
) | ||
|
||
class PydanticBaseEnvConfig(BaseModel): | ||
"""Base class for application configuration. | ||
|
||
Similar to Pydantic's BaseSettings class, but we implement our own method to override from the | ||
environment so that it can be run later, after an instance was constructed.""" | ||
|
||
class PydanticBaseEnvConfig(BaseSettings): | ||
class Config: | ||
env_file = env_file | ||
validate_assignment = True | ||
Comment on lines
-14
to
+18
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could require a prefix option here too to provide some namespacing for the various configs, e.g., have the PostgresConfig environment variables all start with a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. (The implementation on PFML does this) |
||
|
||
def override_from_environment(self, environ, prefix=""): | ||
"""Recursively override field values from the given environment variable mapping.""" | ||
for name, field in self.__fields__.items(): | ||
if field.is_complex(): | ||
# Nested models must be instances of this class too. | ||
getattr(self, name).override_from_environment(environ, prefix=name + "_") | ||
continue | ||
|
||
env_var_name = field.field_info.extra.get("env", prefix + name) | ||
for key in (env_var_name, env_var_name.lower(), env_var_name.upper()): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Might be nice to be stricter here and require a certain casing, avoid letting someone accidentally set the environment variable in lowercase and then not understand why it's not being overridden if it's also set in uppercase elsewhere. |
||
if key in environ: | ||
logging.info("override from environment", extra={"key": key}) | ||
setattr(self, field.name, environ[key]) | ||
break |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This code should move to a unit test, where it checks that each one is an instance of
RootConfig
.