From ae9224ab17214d5c30d7eacb0ba7d5307118e151 Mon Sep 17 00:00:00 2001 From: Nick Schrock Date: Tue, 24 Dec 2024 08:48:38 -0800 Subject: [PATCH] [components] ComponentRegistry --> ComponentTypeRegistry (#26719) ## Summary & Motivation ## How I Tested These Changes ## Changelog > Insert changelog entry or delete this section. --- docs/content/api/modules.json.gz | Bin 1436701 -> 1436701 bytes docs/content/api/searchindex.json.gz | Bin 85946 -> 85946 bytes docs/content/api/sections.json.gz | Bin 484941 -> 484941 bytes .../hello_world/hello_world/definitions.py | 4 +- .../dagster_components/__init__.py | 2 +- .../dagster_components/cli/generate.py | 6 +- .../dagster_components/cli/list.py | 8 +- .../dagster_components/core/component.py | 75 +++++++++--------- .../core/component_defs_builder.py | 18 ++--- .../dagster_components/core/deployment.py | 8 +- .../integration_tests/component_loader.py | 13 +-- .../unit_tests/test_registered_component.py | 16 ++-- .../unit_tests/test_registry.py | 4 +- .../dagster_components_tests/utils.py | 6 +- 14 files changed, 84 insertions(+), 76 deletions(-) diff --git a/docs/content/api/modules.json.gz b/docs/content/api/modules.json.gz index 7ead3e9f0e440ce599059bc7be32f326f8f1ae79..f33820543aeac95bf4e1048ce88f4a269920f3b7 100644 GIT binary patch delta 80 zcmWN_IT3?U06@XvdimV5CL{iD* aom_$xQc5MYG}1~Zy$mwSB%dGO`F;RfXA@)q diff --git a/docs/content/api/searchindex.json.gz b/docs/content/api/searchindex.json.gz index cb9e6e5c4b1aa8f428fc4f4c37fd23e698167109..b6ddb156e9d0c1183473109b93b2bf0b54c0192a 100644 GIT binary patch delta 20 ccmdlrpLN%KRyO%=4vzPG8`-w9F>dh(08zjPZvX%Q delta 20 ccmdlrpLN%KRyO%=4i2xKjci-l7`ONX087&b#sB~S diff --git a/docs/content/api/sections.json.gz b/docs/content/api/sections.json.gz index acd60a9bed3b8300d1f8101629b034309ddbaa9a..329cb44d701981c95107c4e3202e82d9236c17ca 100644 GIT binary patch delta 38 ucmX?mM)vF(SvL7@4vu$w8`)af7+cwxTG^Oe*;rcHSX None: context = CodeLocationProjectContext.from_code_location_path( find_enclosing_code_location_root_path(Path.cwd()), - ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib), + ComponentTypeRegistry.from_entry_point_discovery( + builtin_component_lib=builtin_component_lib + ), ) output: Dict[str, Any] = {} for key, component_type in context.list_component_types(): package, name = key.rsplit(".", 1) - output[key] = ComponentMetadata( + output[key] = ComponentTypeMetadata( name=name, package=package, **component_type.get_metadata(), diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component.py b/python_modules/libraries/dagster-components/dagster_components/core/component.py index 62cb60ba34135..a4e76a3c815a8 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component.py @@ -66,7 +66,7 @@ def build_defs(self, context: "ComponentLoadContext") -> Definitions: ... def load(cls, context: "ComponentLoadContext") -> Self: ... @classmethod - def get_metadata(cls) -> "ComponentInternalMetadata": + def get_metadata(cls) -> "ComponentTypeInternalMetadata": docstring = cls.__doc__ clean_docstring = _clean_docstring(docstring) if docstring else None @@ -104,14 +104,14 @@ def _get_click_cli_help(command: click.Command) -> str: return formatter.getvalue() -class ComponentInternalMetadata(TypedDict): +class ComponentTypeInternalMetadata(TypedDict): summary: Optional[str] description: Optional[str] generate_params_schema: Optional[Any] # json schema component_params_schema: Optional[Any] # json schema -class ComponentMetadata(ComponentInternalMetadata): +class ComponentTypeMetadata(ComponentTypeInternalMetadata): name: str package: str @@ -129,27 +129,28 @@ def get_entry_points_from_python_environment(group: str) -> Sequence[importlib.m BUILTIN_TEST_COMPONENT_ENTRY_POINT = ".".join([BUILTIN_COMPONENTS_ENTRY_POINT_BASE, "test"]) -class ComponentRegistry: +class ComponentTypeRegistry: @classmethod def from_entry_point_discovery( cls, builtin_component_lib: str = BUILTIN_MAIN_COMPONENT_ENTRY_POINT - ) -> "ComponentRegistry": - """Discover components registered in the Python environment via the `dagster_components` entry point group. + ) -> "ComponentTypeRegistry": + """Discover component types registered in the Python environment via the + `dagster_components` entry point group. `dagster-components` itself registers multiple component entry points. We call these "builtin" component libraries. The `dagster_components` entry point resolves to published - components and is loaded by default. Other entry points resolve to various sets of test - components. This method will only ever load one builtin component library. + component types and is loaded by default. Other entry points resolve to various sets of test + component types. This method will only ever load one builtin component library. Args: - builtin-component-lib (str): Specifies the builtin components library to load. Builtin - copmonents libraries are defined under entry points with names matching the pattern - `dagster_components*`. Only one builtin component library can be loaded at a time. - Defaults to `dagster_components`, the standard set of published components. + builtin-component-lib (str): Specifies the builtin components library to load. Built-in + component libraries are defined under entry points with names matching the pattern + `dagster_components*`. Only one built-in component library can be loaded at a time. + Defaults to `dagster_components`, the standard set of published component types. """ - components: Dict[str, Type[Component]] = {} + component_types: Dict[str, Type[Component]] = {} for entry_point in get_entry_points_from_python_environment(COMPONENTS_ENTRY_POINT_GROUP): - # Skip builtin entry points that are not the specified builtin component library. + # Skip built-in entry points that are not the specified builtin component library. if ( entry_point.name.startswith(BUILTIN_COMPONENTS_ENTRY_POINT_BASE) and not entry_point.name == builtin_component_lib @@ -162,44 +163,44 @@ def from_entry_point_discovery( f"Invalid entry point {entry_point.name} in group {COMPONENTS_ENTRY_POINT_GROUP}. " f"Value expected to be a module, got {root_module}." ) - for component in get_registered_components_in_module(root_module): - key = f"{entry_point.name}.{get_component_name(component)}" - components[key] = component + for component_type in get_registered_component_types_in_module(root_module): + key = f"{entry_point.name}.{get_component_type_name(component_type)}" + component_types[key] = component_type - return cls(components) + return cls(component_types) - def __init__(self, components: Dict[str, Type[Component]]): - self._components: Dict[str, Type[Component]] = copy.copy(components) + def __init__(self, component_types: Dict[str, Type[Component]]): + self._component_types: Dict[str, Type[Component]] = copy.copy(component_types) @staticmethod - def empty() -> "ComponentRegistry": - return ComponentRegistry({}) + def empty() -> "ComponentTypeRegistry": + return ComponentTypeRegistry({}) - def register(self, name: str, component: Type[Component]) -> None: - if name in self._components: + def register(self, name: str, component_type: Type[Component]) -> None: + if name in self._component_types: raise DagsterError(f"There is an existing component registered under {name}") - self._components[name] = component + self._component_types[name] = component_type def has(self, name: str) -> bool: - return name in self._components + return name in self._component_types def get(self, name: str) -> Type[Component]: - return self._components[name] + return self._component_types[name] def keys(self) -> Iterable[str]: - return self._components.keys() + return self._component_types.keys() def __repr__(self) -> str: - return f"" + return f"" -def get_registered_components_in_module(module: ModuleType) -> Iterable[Type[Component]]: +def get_registered_component_types_in_module(module: ModuleType) -> Iterable[Type[Component]]: from dagster._core.definitions.module_loaders.load_assets_from_modules import ( find_subclasses_in_module, ) for component in find_subclasses_in_module(module, (Component,)): - if is_registered_component(component): + if is_registered_component_type(component): yield component @@ -209,7 +210,7 @@ def get_registered_components_in_module(module: ModuleType) -> Iterable[Type[Com @dataclass class ComponentLoadContext: resources: Mapping[str, object] - registry: ComponentRegistry + registry: ComponentTypeRegistry decl_node: Optional[ComponentDeclNode] templated_value_resolver: TemplatedValueResolver @@ -217,12 +218,12 @@ class ComponentLoadContext: def for_test( *, resources: Optional[Mapping[str, object]] = None, - registry: Optional[ComponentRegistry] = None, + registry: Optional[ComponentTypeRegistry] = None, decl_node: Optional[ComponentDeclNode] = None, ) -> "ComponentLoadContext": return ComponentLoadContext( resources=resources or {}, - registry=registry or ComponentRegistry.empty(), + registry=registry or ComponentTypeRegistry.empty(), decl_node=decl_node, templated_value_resolver=TemplatedValueResolver.default(), ) @@ -293,13 +294,13 @@ def wrapper(actual_cls: Type[Component]) -> Type[Component]: return cls -def is_registered_component(cls: Type) -> bool: +def is_registered_component_type(cls: Type) -> bool: return hasattr(cls, COMPONENT_REGISTRY_KEY_ATTR) -def get_component_name(component_type: Type[Component]) -> str: +def get_component_type_name(component_type: Type[Component]) -> str: check.param_invariant( - is_registered_component(component_type), + is_registered_component_type(component_type), "component_type", "Expected a registered component. Use @component to register a component.", ) diff --git a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py index e871e1c0a469a..a114cea577cf6 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py @@ -10,10 +10,10 @@ from dagster_components.core.component import ( Component, ComponentLoadContext, - ComponentRegistry, + ComponentTypeRegistry, TemplatedValueResolver, - get_component_name, - is_registered_component, + get_component_type_name, + is_registered_component_type, ) from dagster_components.core.component_decl_builder import ( ComponentFolder, @@ -54,7 +54,7 @@ def load_components_from_context(context: ComponentLoadContext) -> Sequence[Comp def component_type_from_yaml_decl( - registry: ComponentRegistry, decl_node: YamlComponentDecl + registry: ComponentTypeRegistry, decl_node: YamlComponentDecl ) -> Type[Component]: parsed_defs = decl_node.component_file_model if parsed_defs.type.startswith("."): @@ -69,8 +69,8 @@ def component_type_from_yaml_decl( for _name, obj in inspect.getmembers(module, inspect.isclass): assert isinstance(obj, Type) if ( - is_registered_component(obj) - and get_component_name(obj) == component_registry_key + is_registered_component_type(obj) + and get_component_type_name(obj) == component_registry_key ): return obj @@ -91,7 +91,7 @@ def build_components_from_component_folder( def build_defs_from_component_path( path: Path, - registry: ComponentRegistry, + registry: ComponentTypeRegistry, resources: Mapping[str, object], ) -> "Definitions": """Build a definitions object from a folder within the components hierarchy.""" @@ -128,7 +128,7 @@ def defs_from_components( def build_component_defs( code_location_root: Path, resources: Optional[Mapping[str, object]] = None, - registry: Optional["ComponentRegistry"] = None, + registry: Optional["ComponentTypeRegistry"] = None, components_folder: Optional[Path] = None, ) -> "Definitions": """Build a Definitions object for all the component instances in a given code location. @@ -141,7 +141,7 @@ def build_component_defs( context = CodeLocationProjectContext.from_code_location_path( code_location_root, - registry or ComponentRegistry.from_entry_point_discovery(), + registry or ComponentTypeRegistry.from_entry_point_discovery(), components_folder=components_folder, ) diff --git a/python_modules/libraries/dagster-components/dagster_components/core/deployment.py b/python_modules/libraries/dagster-components/dagster_components/core/deployment.py index 3fcff5eaf1142..98c16f73c8400 100644 --- a/python_modules/libraries/dagster-components/dagster_components/core/deployment.py +++ b/python_modules/libraries/dagster-components/dagster_components/core/deployment.py @@ -6,7 +6,7 @@ from dagster._core.errors import DagsterError from typing_extensions import Self -from dagster_components.core.component import Component, ComponentRegistry +from dagster_components.core.component import Component, ComponentTypeRegistry # Code location _CODE_LOCATION_CUSTOM_COMPONENTS_DIR: Final = "lib" @@ -49,7 +49,7 @@ class CodeLocationProjectContext: def from_code_location_path( cls, path: Path, - component_registry: "ComponentRegistry", + component_registry: "ComponentTypeRegistry", components_folder: Optional[Path] = None, ) -> Self: if not _is_code_location_root(path): @@ -71,7 +71,7 @@ def __init__( self, root_path: str, name: str, - component_registry: "ComponentRegistry", + component_registry: "ComponentTypeRegistry", components_folder: Path, ): self._root_path = root_path @@ -88,7 +88,7 @@ def component_types_root_module(self) -> str: return f"{self._name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}" @property - def component_registry(self) -> "ComponentRegistry": + def component_registry(self) -> "ComponentTypeRegistry": return self._component_registry def has_component_type(self, name: str) -> bool: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/component_loader.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/component_loader.py index 88fe3ec52bc06..afb65fc51b665 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/component_loader.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/component_loader.py @@ -2,10 +2,13 @@ from pathlib import Path from dagster._core.definitions.definitions_class import Definitions -from dagster_components.core.component import ComponentRegistry, get_registered_components_in_module +from dagster_components.core.component import ( + ComponentTypeRegistry, + get_registered_component_types_in_module, +) from dagster_components.core.component_defs_builder import ( build_defs_from_component_path, - get_component_name, + get_component_type_name, ) from dagster_components.core.deployment import CodeLocationProjectContext @@ -24,13 +27,13 @@ def load_test_component_project_context() -> CodeLocationProjectContext: dc_module = importlib.import_module(package_name) components = {} - for component in get_registered_components_in_module(dc_module): - key = f"dagster_components.{get_component_name(component)}" + for component in get_registered_component_types_in_module(dc_module): + key = f"dagster_components.{get_component_type_name(component)}" components[key] = component return CodeLocationProjectContext( root_path=str(Path(__file__).parent), name="test", - component_registry=ComponentRegistry(components), + component_registry=ComponentTypeRegistry(components), components_folder=Path(__file__).parent / "components", ) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py index 309959d1d57eb..f53b7dd12038a 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registered_component.py @@ -1,32 +1,32 @@ from dagster_components import Component, component_type -from dagster_components.core.component import get_component_name, is_registered_component +from dagster_components.core.component import get_component_type_name, is_registered_component_type def test_registered_component_with_default_name() -> None: @component_type class RegisteredComponent(Component): ... - assert is_registered_component(RegisteredComponent) - assert get_component_name(RegisteredComponent) == "registered_component" + assert is_registered_component_type(RegisteredComponent) + assert get_component_type_name(RegisteredComponent) == "registered_component" def test_registered_component_with_default_name_and_parens() -> None: @component_type() class RegisteredComponent(Component): ... - assert is_registered_component(RegisteredComponent) - assert get_component_name(RegisteredComponent) == "registered_component" + assert is_registered_component_type(RegisteredComponent) + assert get_component_type_name(RegisteredComponent) == "registered_component" def test_registered_component_with_explicit_kwarg_name() -> None: @component_type(name="explicit_name") class RegisteredComponent(Component): ... - assert is_registered_component(RegisteredComponent) - assert get_component_name(RegisteredComponent) == "explicit_name" + assert is_registered_component_type(RegisteredComponent) + assert get_component_type_name(RegisteredComponent) == "explicit_name" def test_unregistered_component() -> None: class UnregisteredComponent(Component): ... - assert not is_registered_component(UnregisteredComponent) + assert not is_registered_component_type(UnregisteredComponent) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py index dd39dcf743efc..386e07a9a9f1a 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/unit_tests/test_registry.py @@ -26,9 +26,9 @@ def _temp_venv(install_args: Sequence[str]) -> Iterator[Path]: COMPONENT_PRINT_SCRIPT = """ -from dagster_components import ComponentRegistry +from dagster_components import ComponentTypeRegistry -registry = ComponentRegistry.from_entry_point_discovery() +registry = ComponentTypeRegistry.from_entry_point_discovery() for component_name in list(registry.keys()): print(component_name) """ diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/utils.py b/python_modules/libraries/dagster-components/dagster_components_tests/utils.py index 9dd4669a4fe8f..e7f9a72acfcb9 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/utils.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/utils.py @@ -10,12 +10,12 @@ Component, ComponentDeclNode, ComponentLoadContext, - ComponentRegistry, + ComponentTypeRegistry, ) -def registry() -> ComponentRegistry: - return ComponentRegistry.from_entry_point_discovery() +def registry() -> ComponentTypeRegistry: + return ComponentTypeRegistry.from_entry_point_discovery() def script_load_context(decl_node: Optional[ComponentDeclNode] = None) -> ComponentLoadContext: