From c3a4549862fe5f263375224145b0119376360341 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Tue, 10 Dec 2024 09:49:18 -0500 Subject: [PATCH] [components] Add entry point mechanism for component registration (#26337) ## Summary & Motivation Change the `dagster-components` registration system to use Python entry points. All components picked up by the system must be exposed as entry points. This includes: - components exposed by `dagster-components` itself - components exposed by arbitrary third party libraries - components exposed by the code location module An entry point is declared under the group `dagster.components` in `pyproject.toml`/`setup.py`: ``` [project.entry-points] "dagster.components" = { lib = "my_pkg.lib" } ``` A new `ComponentRegistry.from_entry_point_discovery()` method is the preferred way to construct a registry. It will pull all entry points from this group using `importlib.metadata` and crawl the specified modules. ## How I Tested These Changes New unit tests. --- .../hello_world/hello_world/definitions.py | 2 +- .../dagster_components/__init__.py | 16 +-- .../dagster_components/cli/generate.py | 4 +- .../dagster_components/cli/list.py | 3 +- .../dagster_components/core/component.py | 41 +++++- .../core/component_defs_builder.py | 11 +- .../dagster_components/core/deployment.py | 19 +-- .../{impls => lib}/__init__.py | 0 .../{impls => lib}/dbt_project.py | 0 .../pipes_subprocess_script_collection.py | 8 +- .../{impls => lib}/sling_replication.py | 0 .../debug_sling_component/component.py | 2 +- .../components/scripts/component.yaml | 2 +- .../integration_tests/test_dbt_project.py | 2 +- .../test_sling_integration_test.py | 2 +- .../test_templated_custom_keys_dbt_project.py | 2 +- .../components/jaffle_shop_dbt/component.yaml | 2 +- .../components/ingest/component.yaml | 2 +- .../components/jaffle_shop_dbt/component.yaml | 2 +- ...test_pipes_subprocess_script_collection.py | 2 +- .../unit_tests/test_registered_component.py | 10 +- .../unit_tests/test_registry.py | 119 ++++++++++++++++++ .../dagster_components_tests/utils.py | 3 +- .../libraries/dagster-components/setup.py | 5 +- .../libraries/dg-cli/dg_cli/cli/generate.py | 3 +- .../libraries/dg-cli/dg_cli/py.typed | 0 .../pyproject.toml.jinja | 3 + .../cli_tests/test_generate_commands.py | 18 +-- 28 files changed, 209 insertions(+), 74 deletions(-) rename python_modules/libraries/dagster-components/dagster_components/{impls => lib}/__init__.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{impls => lib}/dbt_project.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{impls => lib}/pipes_subprocess_script_collection.py (95%) rename python_modules/libraries/dagster-components/dagster_components/{impls => lib}/sling_replication.py (100%) create mode 100644 python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py create mode 100644 python_modules/libraries/dg-cli/dg_cli/py.typed diff --git a/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py b/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py index 46f9002ec61e6..b7a4a42f65ee3 100644 --- a/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py +++ b/examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py @@ -2,7 +2,7 @@ import dagster as dg from dagster_components import ComponentRegistry, build_defs_from_toplevel_components_folder -from dagster_components.impls.pipes_subprocess_script_collection import ( +from dagster_components.lib.pipes_subprocess_script_collection import ( PipesSubprocessScriptCollection, ) diff --git a/python_modules/libraries/dagster-components/dagster_components/__init__.py b/python_modules/libraries/dagster-components/dagster_components/__init__.py index f83bc23e5269a..f5463bf0d60f8 100644 --- a/python_modules/libraries/dagster-components/dagster_components/__init__.py +++ b/python_modules/libraries/dagster-components/dagster_components/__init__.py @@ -1,3 +1,5 @@ +from dagster._core.libraries import DagsterLibraryRegistry + from dagster_components.core.component import ( Component as Component, ComponentLoadContext as ComponentLoadContext, @@ -7,20 +9,6 @@ from dagster_components.core.component_defs_builder import ( build_defs_from_toplevel_components_folder as build_defs_from_toplevel_components_folder, ) -from dagster_components.impls.dbt_project import DbtProjectComponent -from dagster_components.impls.pipes_subprocess_script_collection import ( - PipesSubprocessScriptCollection, -) -from dagster_components.impls.sling_replication import SlingReplicationComponent - -__component_registry__ = { - "pipes_subprocess_script_collection": PipesSubprocessScriptCollection, - "sling_replication": SlingReplicationComponent, - "dbt_project": DbtProjectComponent, -} - -from dagster._core.libraries import DagsterLibraryRegistry - from dagster_components.version import __version__ as __version__ DagsterLibraryRegistry.register("dagster-components", __version__) diff --git a/python_modules/libraries/dagster-components/dagster_components/cli/generate.py b/python_modules/libraries/dagster-components/dagster_components/cli/generate.py index 1c77fa6a0d033..48d476736a483 100644 --- a/python_modules/libraries/dagster-components/dagster_components/cli/generate.py +++ b/python_modules/libraries/dagster-components/dagster_components/cli/generate.py @@ -5,7 +5,7 @@ import click from pydantic import TypeAdapter -from dagster_components import ComponentRegistry, __component_registry__ +from dagster_components import ComponentRegistry from dagster_components.core.deployment import ( CodeLocationProjectContext, is_inside_code_location_project, @@ -38,7 +38,7 @@ def generate_component_command( sys.exit(1) context = CodeLocationProjectContext.from_path( - Path.cwd(), ComponentRegistry(__component_registry__) + Path.cwd(), ComponentRegistry.from_entry_point_discovery() ) if not context.has_component_type(component_type): click.echo( diff --git a/python_modules/libraries/dagster-components/dagster_components/cli/list.py b/python_modules/libraries/dagster-components/dagster_components/cli/list.py index 1d2bed67da213..9368a9fa1206a 100644 --- a/python_modules/libraries/dagster-components/dagster_components/cli/list.py +++ b/python_modules/libraries/dagster-components/dagster_components/cli/list.py @@ -5,7 +5,6 @@ import click -from dagster_components import __component_registry__ from dagster_components.core.component import ComponentRegistry from dagster_components.core.deployment import ( CodeLocationProjectContext, @@ -30,7 +29,7 @@ def list_component_types_command() -> None: sys.exit(1) context = CodeLocationProjectContext.from_path( - Path.cwd(), ComponentRegistry(__component_registry__) + Path.cwd(), ComponentRegistry.from_entry_point_discovery() ) output: Dict[str, Any] = {} for component_type in context.list_component_types(): diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component.py b/python_modules/libraries/dagster-components/dagster_components/core/component.py index 2d27bc50ba9b0..fe15ed559e48e 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component.py @@ -1,7 +1,10 @@ import copy +import importlib +import importlib.metadata +import sys from abc import ABC, abstractmethod from types import ModuleType -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, Mapping, Optional, Type +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, Mapping, Optional, Sequence, Type from dagster import _check as check from dagster._core.errors import DagsterError @@ -33,7 +36,33 @@ def from_decl_node( ) -> Self: ... +def get_entry_points_from_python_environment(group: str) -> Sequence[importlib.metadata.EntryPoint]: + if sys.version_info >= (3, 10): + return importlib.metadata.entry_points(group=group) + else: + return importlib.metadata.entry_points().get(group, []) + + +COMPONENTS_ENTRY_POINT_GROUP = "dagster.components" + + class ComponentRegistry: + @classmethod + def from_entry_point_discovery(cls) -> "ComponentRegistry": + components: Dict[str, Type[Component]] = {} + for entry_point in get_entry_points_from_python_environment(COMPONENTS_ENTRY_POINT_GROUP): + root_module = entry_point.load() + if not isinstance(root_module, ModuleType): + raise DagsterError( + f"Invalid entry point {entry_point.name} in group {COMPONENTS_ENTRY_POINT_GROUP}. " + f"Value expected to be a module, got {root_module}." + ) + for component in get_registered_components_in_module(root_module): + key = f"{entry_point.name}.{get_component_name(component)}" + components[key] = component + + return cls(components) + def __init__(self, components: Dict[str, Type[Component]]): self._components: Dict[str, Type[Component]] = copy.copy(components) @@ -59,7 +88,7 @@ def __repr__(self) -> str: return f"" -def register_components_in_module(registry: ComponentRegistry, root_module: ModuleType) -> None: +def get_registered_components_in_module(root_module: ModuleType) -> Iterable[Type[Component]]: from dagster._core.definitions.load_assets_from_modules import ( find_modules_in_package, find_subclasses_in_module, @@ -67,8 +96,8 @@ def register_components_in_module(registry: ComponentRegistry, root_module: Modu for module in find_modules_in_package(root_module): for component in find_subclasses_in_module(module, (Component,)): - if is_component(component): - registry.register(get_component_name(component), component) + if is_registered_component(component): + yield component class ComponentLoadContext: @@ -114,13 +143,13 @@ def wrapper(actual_cls: Type[Component]) -> Type[Component]: return cls -def is_component(cls: Type) -> bool: +def is_registered_component(cls: Type) -> bool: return hasattr(cls, COMPONENT_REGISTRY_KEY_ATTR) def get_component_name(component_type: Type[Component]) -> str: check.param_invariant( - is_component(component_type), + is_registered_component(component_type), "component_type", "Expected a registered component. Use @component to register a component.", ) diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py index b4bca32ff4852..3d3e7ab4d7851 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py @@ -13,7 +13,7 @@ ComponentLoadContext, ComponentRegistry, get_component_name, - is_component, + is_registered_component, ) from dagster_components.core.component_decl_builder import ( ComponentFolder, @@ -69,7 +69,10 @@ def component_type_from_yaml_decl( for _name, obj in inspect.getmembers(module, inspect.isclass): assert isinstance(obj, Type) - if is_component(obj) and get_component_name(obj) == component_registry_key: + if ( + is_registered_component(obj) + and get_component_name(obj) == component_registry_key + ): return obj raise Exception( @@ -127,10 +130,8 @@ def build_defs_from_toplevel_components_folder( """Build a Definitions object from an entire component hierarchy.""" from dagster._core.definitions.definitions_class import Definitions - from dagster_components import __component_registry__ - context = CodeLocationProjectContext.from_path( - path, registry or ComponentRegistry(__component_registry__) + path, registry or ComponentRegistry.from_entry_point_discovery() ) all_defs: List[Definitions] = [] diff --git a/python_modules/libraries/dagster-components/dagster_components/core/deployment.py b/python_modules/libraries/dagster-components/dagster_components/core/deployment.py index 59b6f762a59be..006c0f2cdb2c3 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/deployment.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/deployment.py @@ -1,6 +1,4 @@ -import importlib.util import os -import sys from pathlib import Path from typing import Final, Iterable, Type @@ -8,11 +6,7 @@ from dagster._core.errors import DagsterError from typing_extensions import Self -from dagster_components.core.component import ( - Component, - ComponentRegistry, - register_components_in_module, -) +from dagster_components.core.component import Component, ComponentRegistry # Code location _CODE_LOCATION_CUSTOM_COMPONENTS_DIR: Final = "lib" @@ -48,17 +42,6 @@ class CodeLocationProjectContext: @classmethod def from_path(cls, path: Path, component_registry: "ComponentRegistry") -> Self: root_path = _resolve_code_location_root_path(path) - name = os.path.basename(root_path) - - # TODO: Rm when a more robust solution is implemented - # Make sure we can import from the cwd - if sys.path[0] != "": - sys.path.insert(0, "") - - components_lib_module = f"{name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}" - module = importlib.import_module(components_lib_module) - register_components_in_module(component_registry, module) - return cls( root_path=str(root_path), name=os.path.basename(root_path), diff --git a/python_modules/libraries/dagster-components/dagster_components/impls/__init__.py b/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/impls/__init__.py rename to python_modules/libraries/dagster-components/dagster_components/lib/__init__.py diff --git a/python_modules/libraries/dagster-components/dagster_components/impls/dbt_project.py b/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/impls/dbt_project.py rename to python_modules/libraries/dagster-components/dagster_components/lib/dbt_project.py diff --git a/python_modules/libraries/dagster-components/dagster_components/impls/pipes_subprocess_script_collection.py b/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py similarity index 95% rename from python_modules/libraries/dagster-components/dagster_components/impls/pipes_subprocess_script_collection.py rename to python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py index 5c948c65cc7e0..cc461b6b2b234 100644 --- a/python_modules/libraries/dagster-components/dagster_components/impls/pipes_subprocess_script_collection.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py @@ -11,7 +11,12 @@ from dagster._utils.warnings import suppress_dagster_warnings from pydantic import BaseModel, TypeAdapter -from dagster_components.core.component import Component, ComponentDeclNode, ComponentLoadContext +from dagster_components.core.component import ( + Component, + ComponentDeclNode, + ComponentLoadContext, + component, +) from dagster_components.core.component_decl_builder import YamlComponentDecl if TYPE_CHECKING: @@ -48,6 +53,7 @@ class PipesSubprocessScriptCollectionParams(BaseModel): scripts: Sequence[PipesSubprocessScriptParams] +@component(name="pipes_subprocess_script_collection") class PipesSubprocessScriptCollection(Component): params_schema = PipesSubprocessScriptCollectionParams diff --git a/python_modules/libraries/dagster-components/dagster_components/impls/sling_replication.py b/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/impls/sling_replication.py rename to python_modules/libraries/dagster-components/dagster_components/lib/sling_replication.py diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/components/debug_sling_component/component.py b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/components/debug_sling_component/component.py index a69649d5a8c92..0019f56f18974 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/components/debug_sling_component/component.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/components/debug_sling_component/component.py @@ -2,7 +2,7 @@ from dagster._core.execution.context.asset_execution_context import AssetExecutionContext from dagster_components import component -from dagster_components.impls.sling_replication import SlingReplicationComponent +from dagster_components.lib.sling_replication import SlingReplicationComponent from dagster_embedded_elt.sling import SlingResource diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml index 5699c0df48030..34bd590951b8b 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml @@ -1,4 +1,4 @@ -type: pipes_subprocess_script_collection +type: dagster_components.pipes_subprocess_script_collection params: scripts: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py index 5386816b507b4..d3bc69c881f32 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py @@ -12,7 +12,7 @@ build_components_from_component_folder, defs_from_components, ) -from dagster_components.impls.dbt_project import DbtProjectComponent +from dagster_components.lib.dbt_project import DbtProjectComponent from dagster_dbt import DbtProject from dagster_components_tests.utils import assert_assets, get_asset_keys, script_load_context diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py index 2b77ece55b986..494e4255aa853 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py @@ -16,7 +16,7 @@ YamlComponentDecl, build_components_from_component_folder, ) -from dagster_components.impls.sling_replication import SlingReplicationComponent, component +from dagster_components.lib.sling_replication import SlingReplicationComponent, component from dagster_embedded_elt.sling import SlingResource from dagster_components_tests.utils import assert_assets, get_asset_keys, script_load_context diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py index 933d2e608fd77..5b21bed30e642 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py @@ -12,7 +12,7 @@ build_components_from_component_folder, defs_from_components, ) -from dagster_components.impls.dbt_project import DbtProjectComponent +from dagster_components.lib.dbt_project import DbtProjectComponent from dagster_dbt import DbtProject from dagster_components_tests.utils import assert_assets, get_asset_keys, script_load_context diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml index 4ea38b095dc78..85c04ada2fb8d 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml @@ -1,4 +1,4 @@ -type: dbt_project +type: dagster_components.dbt_project params: dbt: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/sling_location/components/ingest/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/sling_location/components/ingest/component.yaml index 87ae5b8cdd0ab..e92d6ec0ad839 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/sling_location/components/ingest/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/sling_location/components/ingest/component.yaml @@ -1,4 +1,4 @@ -type: sling_replication +type: dagster_components.sling_replication params: sling: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml index f5a2739845897..07a7bfff6f501 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/stub_code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml @@ -1,4 +1,4 @@ -type: dbt_project +type: dagster_components.dbt_project params: dbt: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_pipes_subprocess_script_collection.py b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_pipes_subprocess_script_collection.py index 796aa2154fe7e..ae4206ef1fe58 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_pipes_subprocess_script_collection.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_pipes_subprocess_script_collection.py @@ -8,7 +8,7 @@ build_defs_from_component_path, defs_from_components, ) -from dagster_components.impls.pipes_subprocess_script_collection import ( +from dagster_components.lib.pipes_subprocess_script_collection import ( PipesSubprocessScriptCollection, ) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py index 4caccde535422..bf7fddb6423f4 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py @@ -1,12 +1,12 @@ from dagster_components import Component, component -from dagster_components.core.component import get_component_name, is_component +from dagster_components.core.component import get_component_name, is_registered_component def test_registered_component_with_default_name() -> None: @component class RegisteredComponent(Component): ... - assert is_component(RegisteredComponent) + assert is_registered_component(RegisteredComponent) assert get_component_name(RegisteredComponent) == "registered_component" @@ -14,7 +14,7 @@ def test_registered_component_with_default_name_and_parens() -> None: @component() class RegisteredComponent(Component): ... - assert is_component(RegisteredComponent) + assert is_registered_component(RegisteredComponent) assert get_component_name(RegisteredComponent) == "registered_component" @@ -22,11 +22,11 @@ def test_registered_component_with_explicit_kwarg_name() -> None: @component(name="explicit_name") class RegisteredComponent(Component): ... - assert is_component(RegisteredComponent) + assert is_registered_component(RegisteredComponent) assert get_component_name(RegisteredComponent) == "explicit_name" def test_unregistered_component() -> None: class UnregisteredComponent(Component): ... - assert not is_component(UnregisteredComponent) + assert not is_registered_component(UnregisteredComponent) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py new file mode 100644 index 0000000000000..0c52535b44096 --- /dev/null +++ b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py @@ -0,0 +1,119 @@ +import os +import subprocess +import sys +from pathlib import Path + +from dagster_components import ComponentRegistry + + +def test_components_from_dagster(): + registry = ComponentRegistry.from_entry_point_discovery() + assert registry.has("dagster_components.dbt_project") + assert registry.has("dagster_components.sling_replication") + assert registry.has("dagster_components.pipes_subprocess_script_collection") + + +def _find_repo_root(): + current = Path(__file__).parent + while not (current / ".git").exists(): + if current == Path("/"): + raise Exception("Could not find the repository root.") + current = current.parent + return current + + +repo_root = _find_repo_root() + +# Our pyproject.toml installs local dagster components +PYPROJECT_TOML = f""" +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dagster-foo" +version = "0.1.0" +description = "A simple example package" +authors = [ + {{ name = "Your Name", email = "your.email@example.com" }} +] +dependencies = [ + "dagster", + "dagster-components", + "dagster-dbt", + "dagster-embedded-elt", +] + +[tool.uv.sources] +dagster = {{ path = "{repo_root}/python_modules/dagster" }} +dagster-pipes = {{ path = "{repo_root}/python_modules/dagster-pipes" }} +dagster-components = {{ path = "{repo_root}/python_modules/libraries/dagster-components" }} +dagster-dbt = {{ path = "{repo_root}/python_modules/libraries/dagster-dbt" }} +dagster-embedded-elt = {{ path = "{repo_root}/python_modules/libraries/dagster-embedded-elt" }} + +[project.entry-points] +"dagster.components" = {{ dagster_foo = "dagster_foo.lib"}} +""" + +TEST_COMPONENT_1 = """ +from dagster_components import Component, component + +@component(name="test_component_1") +class TestComponent1(Component): + pass +""" + +TEST_COMPONENT_2 = """ +from dagster_components import Component, component + +@component(name="test_component_2") +class TestComponent2(Component): + pass +""" + +COMPONENT_PRINT_SCRIPT = """ +from dagster_components import ComponentRegistry + +registry = ComponentRegistry.from_entry_point_discovery() +for component_name in list(registry.keys()): + print(component_name) +""" + + +def test_components_from_third_party_lib(tmpdir): + with tmpdir.as_cwd(): + # Create test package that defines some components + os.makedirs("dagster-foo") + with open("dagster-foo/pyproject.toml", "w") as f: + f.write(PYPROJECT_TOML) + + os.makedirs("dagster-foo/dagster_foo/lib/sub") + + with open("dagster-foo/dagster_foo/lib/__init__.py", "w") as f: + f.write(TEST_COMPONENT_1) + + with open("dagster-foo/dagster_foo/lib/sub/__init__.py", "w") as f: + f.write(TEST_COMPONENT_2) + + # Create venv + venv_dir = Path(".venv") + subprocess.check_call(["uv", "venv", str(venv_dir)]) + python_executable = ( + venv_dir + / ("Scripts" if sys.platform == "win32" else "bin") + / ("python.exe" if sys.platform == "win32" else "python") + ) + + # Script to print components + with open("print_components.py", "w") as f: + f.write(COMPONENT_PRINT_SCRIPT) + + # subprocess.check_call([pip_executable, "install", "-e", "dagster-foo"]) + subprocess.check_call( + ["uv", "pip", "install", "--python", str(python_executable), "-e", "dagster-foo"] + ) + result = subprocess.run( + [python_executable, "print_components.py"], capture_output=True, text=True, check=False + ) + assert "dagster_foo.test_component_1" in result.stdout + assert "dagster_foo.test_component_2" in result.stdout diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/utils.py b/python_modules/libraries/dagster-components/dagster_components_tests/utils.py index a6655490e423e..51bce9a14ea32 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/utils.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/utils.py @@ -1,10 +1,9 @@ from dagster import AssetKey, DagsterInstance -from dagster_components import __component_registry__ from dagster_components.core.component import Component, ComponentLoadContext, ComponentRegistry def registry() -> ComponentRegistry: - return ComponentRegistry(__component_registry__) + return ComponentRegistry.from_entry_point_discovery() def script_load_context() -> ComponentLoadContext: diff --git a/python_modules/libraries/dagster-components/setup.py b/python_modules/libraries/dagster-components/setup.py index e0776f8337fb9..903e3fa1c9f7b 100644 --- a/python_modules/libraries/dagster-components/setup.py +++ b/python_modules/libraries/dagster-components/setup.py @@ -43,7 +43,10 @@ def get_version() -> str: entry_points={ "console_scripts": [ "dagster-components = dagster_components.cli:main", - ] + ], + "dagster.components": [ + "dagster_components = dagster_components.lib", + ], }, extras_require={ "sling": ["dagster-embedded-elt"], diff --git a/python_modules/libraries/dg-cli/dg_cli/cli/generate.py b/python_modules/libraries/dg-cli/dg_cli/cli/generate.py index 69bf22d40b177..0f0da7bd0033d 100644 --- a/python_modules/libraries/dg-cli/dg_cli/cli/generate.py +++ b/python_modules/libraries/dg-cli/dg_cli/cli/generate.py @@ -80,7 +80,8 @@ def generate_component_type_command(name: str) -> None: ) sys.exit(1) context = CodeLocationProjectContext.from_path(Path.cwd()) - if context.has_component_type(name): + full_component_name = f"{context.name}.{name}" + if context.has_component_type(full_component_name): click.echo(click.style(f"A component type named `{name}` already exists.", fg="red")) sys.exit(1) diff --git a/python_modules/libraries/dg-cli/dg_cli/py.typed b/python_modules/libraries/dg-cli/dg_cli/py.typed new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/libraries/dg-cli/dg_cli/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja b/python_modules/libraries/dg-cli/dg_cli/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja index c0022f043f2b4..27c282a79b3c0 100644 --- a/python_modules/libraries/dg-cli/dg_cli/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja +++ b/python_modules/libraries/dg-cli/dg_cli/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja @@ -15,6 +15,9 @@ dev = [ "dagster-webserver", ] +[project.entry-points] +"dagster.components" = { {{ project_name }} = "{{ project_name }}.lib"} + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" diff --git a/python_modules/libraries/dg-cli/dg_cli_tests/cli_tests/test_generate_commands.py b/python_modules/libraries/dg-cli/dg_cli_tests/cli_tests/test_generate_commands.py index c8f776c002881..71bb23d4aa169 100644 --- a/python_modules/libraries/dg-cli/dg_cli_tests/cli_tests/test_generate_commands.py +++ b/python_modules/libraries/dg-cli/dg_cli_tests/cli_tests/test_generate_commands.py @@ -217,7 +217,7 @@ def test_generate_component_type_success(in_deployment: bool) -> None: assert result.exit_code == 0 assert Path("bar/lib/baz.py").exists() context = CodeLocationProjectContext.from_path(Path.cwd()) - assert context.has_component_type("baz") + assert context.has_component_type("bar.baz") def test_generate_component_type_outside_code_location_fails() -> None: @@ -243,7 +243,7 @@ def test_generate_component_type_already_exists_fails(in_deployment: bool) -> No def test_generate_component_success(in_deployment: bool) -> None: runner = CliRunner() with isolated_example_code_location_bar_with_component_type_baz(runner, in_deployment): - result = runner.invoke(generate_component_command, ["baz", "qux"]) + result = runner.invoke(generate_component_command, ["bar.baz", "qux"]) assert result.exit_code == 0 assert Path("bar/components/qux").exists() assert Path("bar/components/qux/sample.py").exists() @@ -252,7 +252,7 @@ def test_generate_component_success(in_deployment: bool) -> None: def test_generate_component_outside_code_location_fails() -> None: runner = CliRunner() with isolated_example_deployment_foo(runner): - result = runner.invoke(generate_component_command, ["baz", "qux"]) + result = runner.invoke(generate_component_command, ["bar.baz", "qux"]) assert result.exit_code != 0 assert "must be run inside a Dagster code location project" in result.output @@ -261,9 +261,9 @@ def test_generate_component_outside_code_location_fails() -> None: def test_generate_component_already_exists_fails(in_deployment: bool) -> None: runner = CliRunner() with isolated_example_code_location_bar_with_component_type_baz(runner, in_deployment): - result = runner.invoke(generate_component_command, ["baz", "qux"]) + result = runner.invoke(generate_component_command, ["bar.baz", "qux"]) assert result.exit_code == 0 - result = runner.invoke(generate_component_command, ["baz", "qux"]) + result = runner.invoke(generate_component_command, ["bar.baz", "qux"]) assert result.exit_code != 0 assert "already exists" in result.output @@ -271,7 +271,9 @@ def test_generate_component_already_exists_fails(in_deployment: bool) -> None: def test_generate_sling_replication_instance() -> None: runner = CliRunner() with isolated_example_code_location_bar(runner): - result = runner.invoke(generate_component_command, ["sling_replication", "file_ingest"]) + result = runner.invoke( + generate_component_command, ["dagster_components.sling_replication", "file_ingest"] + ) assert result.exit_code == 0 assert Path("bar/components/file_ingest").exists() @@ -297,7 +299,9 @@ def test_generate_sling_replication_instance() -> None: def test_generate_dbt_project_instance(params) -> None: runner = CliRunner() with isolated_example_code_location_bar(runner): - result = runner.invoke(generate_component_command, ["dbt_project", "my_project", *params]) + result = runner.invoke( + generate_component_command, ["dagster_components.dbt_project", "my_project", *params] + ) assert result.exit_code == 0 assert Path("bar/components/my_project").exists()