From 08a67bacb398ee2bbe8d88e5577a664570562664 Mon Sep 17 00:00:00 2001 From: Maxime Armstrong <46797220+maximearmstrong@users.noreply.github.com> Date: Mon, 3 Feb 2025 10:28:02 -0500 Subject: [PATCH] [dagster-dbt] Support profiles_dir and profile in DbtProject (#27416) ## Summary & Motivation Same as #21108 but for `profiles_dir` and `profile`. In response to PR #26928, fixes issue #26504 ## How I Tested These Changes Additional tests with BK ## Changelog [dagster-dbt] Specifying a dbt profiles directory and profile is now supported in `DbtProject`. --- .../dagster-dbt/dagster_dbt/core/resource.py | 6 +++ .../dagster-dbt/dagster_dbt/dbt_project.py | 18 ++++++++ .../dagster-dbt/dagster_dbt/errors.py | 4 ++ .../dagster_dbt_tests/core/test_resource.py | 44 ++++++++++++++++++- 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py index 9287adc25f880..8e81880c4d00b 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py @@ -217,6 +217,12 @@ def __init__( if not state_path and project_dir.state_path: state_path = project_dir.state_path + if not profiles_dir and project_dir.profiles_dir: + profiles_dir = project_dir.profiles_dir + + if not profile and project_dir.profile: + profile = project_dir.profile + if not target and project_dir.target: target = project_dir.target diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/dbt_project.py b/python_modules/libraries/dagster-dbt/dagster_dbt/dbt_project.py index 60d3bf34f8f0d..781fa324281b7 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/dbt_project.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/dbt_project.py @@ -11,6 +11,7 @@ from dagster_dbt.errors import ( DagsterDbtManifestNotFoundError, + DagsterDbtProfilesDirectoryNotFoundError, DagsterDbtProjectNotFoundError, DagsterDbtProjectYmlFileNotFoundError, ) @@ -153,6 +154,11 @@ class DbtProject(IHaveNew): The path, relative to the project directory, to output artifacts. It corresponds to the target path in dbt. Default: "target" + profiles_dir (Union[str, Path]): + The path to the directory containing your dbt `profiles.yml`. + By default, the current working directory is used, which is the dbt project directory. + profile (Optional[str]): + The profile from your dbt `profiles.yml` to use for execution, if it should be explicitly set. target (Optional[str]): The target from your dbt `profiles.yml` to use for execution, if it should be explicitly set. packaged_project_dir (Optional[Union[str, Path]]): @@ -203,6 +209,8 @@ def get_env(): name: str project_dir: Path target_path: Path + profiles_dir: Path + profile: Optional[str] target: Optional[str] manifest_path: Path packaged_project_dir: Optional[Path] @@ -215,6 +223,8 @@ def __new__( project_dir: Union[Path, str], *, target_path: Union[Path, str] = Path("target"), + profiles_dir: Optional[Union[Path, str]] = None, + profile: Optional[str] = None, target: Optional[str] = None, packaged_project_dir: Optional[Union[Path, str]] = None, state_path: Optional[Union[Path, str]] = None, @@ -223,6 +233,12 @@ def __new__( if not project_dir.exists(): raise DagsterDbtProjectNotFoundError(f"project_dir {project_dir} does not exist.") + profiles_dir = Path(profiles_dir) if profiles_dir else project_dir + if not profiles_dir.exists(): + raise DagsterDbtProfilesDirectoryNotFoundError( + f"profiles {profiles_dir} does not exist." + ) + packaged_project_dir = Path(packaged_project_dir) if packaged_project_dir else None if not using_dagster_dev() and packaged_project_dir and packaged_project_dir.exists(): project_dir = packaged_project_dir @@ -255,6 +271,8 @@ def __new__( name=dbt_project_yml["name"], project_dir=project_dir, target_path=target_path, + profiles_dir=profiles_dir, + profile=profile, target=target, manifest_path=manifest_path, state_path=project_dir.joinpath(state_path) if state_path else None, diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/errors.py b/python_modules/libraries/dagster-dbt/dagster_dbt/errors.py index 3290b5bc4d316..b33801db852d9 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/errors.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/errors.py @@ -19,6 +19,10 @@ class DagsterDbtProjectNotFoundError(DagsterDbtError): """Error when the specified project directory can not be found.""" +class DagsterDbtProfilesDirectoryNotFoundError(DagsterDbtError): + """Error when the specified profiles directory can not be found.""" + + class DagsterDbtManifestNotFoundError(DagsterDbtError): """Error when we expect manifest.json to generated already but it is absent.""" diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resource.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resource.py index 243eaef72fe96..d4edaccba6839 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resource.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_resource.py @@ -16,7 +16,7 @@ from dagster_dbt.core.resource import DbtCliResource from dagster_dbt.dagster_dbt_translator import DagsterDbtTranslator, DagsterDbtTranslatorSettings from dagster_dbt.dbt_project import DbtProject -from dagster_dbt.errors import DagsterDbtCliRuntimeError +from dagster_dbt.errors import DagsterDbtCliRuntimeError, DagsterDbtProfilesDirectoryNotFoundError from dbt.version import __version__ as dbt_version from packaging import version from pydantic import ValidationError @@ -232,6 +232,26 @@ def test_dbt_profile_configuration() -> None: ] assert dbt_cli_invocation.is_successful() + dbt_cli_invocation = ( + DbtCliResource( + project_dir=DbtProject( + os.fspath(test_jaffle_shop_path), profile="jaffle_shop", target="dev" + ) + ) + .cli(["parse"]) + .wait() + ) + + assert dbt_cli_invocation.process.args == [ + "dbt", + "parse", + "--profile", + "jaffle_shop", + "--target", + "dev", + ] + assert dbt_cli_invocation.is_successful() + @pytest.mark.parametrize( "profiles_dir", [None, test_jaffle_shop_path, os.fspath(test_jaffle_shop_path)] @@ -246,10 +266,24 @@ def test_dbt_profiles_dir_configuration(profiles_dir: Union[str, Path]) -> None: .is_successful() ) + assert ( + DbtCliResource( + project_dir=DbtProject(os.fspath(test_jaffle_shop_path), profiles_dir=profiles_dir) + ) + .cli(["parse"]) + .is_successful() + ) + # profiles directory must exist with pytest.raises(ValidationError, match="does not exist"): DbtCliResource(project_dir=os.fspath(test_jaffle_shop_path), profiles_dir="nonexistent") + # Error is raised at the DbtProject level when a nonexistent directory is passed to a DbtProject object + with pytest.raises(DagsterDbtProfilesDirectoryNotFoundError, match="does not exist"): + DbtCliResource( + project_dir=DbtProject(os.fspath(test_jaffle_shop_path), profiles_dir="nonexistent") + ) + # profiles directory must contain profile configuration with pytest.raises(ValidationError, match="specify a valid path to a dbt profile directory"): DbtCliResource( @@ -257,6 +291,14 @@ def test_dbt_profiles_dir_configuration(profiles_dir: Union[str, Path]) -> None: profiles_dir=f"{os.fspath(test_jaffle_shop_path)}/models", ) + with pytest.raises(ValidationError, match="specify a valid path to a dbt profile directory"): + DbtCliResource( + project_dir=DbtProject( + os.fspath(test_jaffle_shop_path), + profiles_dir=f"{os.fspath(test_jaffle_shop_path)}/models", + ) + ) + def test_dbt_project_dir_conflicting_env_var(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DBT_PROJECT_DIR", "nonexistent")