From 941930c4c600d70d82acf80b7f0b23b22f57055e Mon Sep 17 00:00:00 2001 From: gibsondan Date: Tue, 10 Sep 2024 10:58:04 -0500 Subject: [PATCH] Gate duckdb-specific instance check in dagster-dbt in a way that doesn't require an import of a module that might not be present (#24346) Summary: Attempt to resolve https://github.com/dagster-io/dagster/issues/23952: Instead of attempting to import duckdb and logging a warning on importerror, do a bit of introspection on the relevant class and only try to import it if the class name matches. Test Plan: could use some help with this from reviewers, I don't see any duckdb-specific tests in https://github.com/dagster-io/dagster/pull/21542 ## Summary & Motivation ## How I Tested These Changes ## Changelog Insert changelog entry or "NOCHANGELOG" here. - [ ] `NEW` _(added new feature or capability)_ - [ ] `BUGFIX` _(fixed a bug)_ - [ ] `DOCS` _(added or updated documentation)_ --- .../dagster-dbt/dagster_dbt/core/dbt_event_iterator.py | 8 ++++---- .../libraries/dagster-dbt/dagster_dbt/core/resource.py | 9 ++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/core/dbt_event_iterator.py b/python_modules/libraries/dagster-dbt/dagster_dbt/core/dbt_event_iterator.py index 90cbe50a1598b..e6eafb13f8129 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/core/dbt_event_iterator.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/core/dbt_event_iterator.py @@ -277,15 +277,15 @@ def _map_fn(event: DbtDagsterEventType) -> DbtDagsterEventType: # opening multiple connections to the same database when a write connection, such # as the one dbt uses, is open. event_stream = self - try: + if ( + self._dbt_cli_invocation.adapter + and self._dbt_cli_invocation.adapter.__class__.__name__ == "DuckDBAdapter" + ): from dbt.adapters.duckdb import DuckDBAdapter if isinstance(self._dbt_cli_invocation.adapter, DuckDBAdapter): event_stream = exhaust_iterator_and_yield_results_with_exception(self) - except ImportError: - pass - def _threadpool_wrap_map_fn() -> ( Iterator[Union[Output, AssetMaterialization, AssetObservation, AssetCheckResult]] ): diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py index 827ba6c6facd9..c19e44274a1ca 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/core/resource.py @@ -379,7 +379,8 @@ def _initialize_adapter(self, cli_vars) -> BaseAdapter: # If the dbt adapter is DuckDB, set the access mode to READ_ONLY, since DuckDB only allows # simultaneous connections for read-only access. - try: + + if config.credentials and config.credentials.__class__.__name__ == "DuckDBCredentials": from dbt.adapters.duckdb.credentials import DuckDBCredentials if isinstance(config.credentials, DuckDBCredentials): @@ -391,12 +392,6 @@ def _initialize_adapter(self, cli_vars) -> BaseAdapter: with pushd(self.project_dir): config.credentials.path = os.fspath(Path(config.credentials.path).absolute()) - except ImportError: - logger.warning( - "An error was encountered when creating a handle to the dbt adapter in Dagster.", - exc_info=True, - ) - cleanup_event_logger() # reset adapters list in case we have instantiated an adapter before in this process