Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[components] ComponentRegistry --> ComponentTypeRegistry #26719

Merged
merged 5 commits into from
Dec 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified docs/content/api/modules.json.gz
Binary file not shown.
Binary file modified docs/content/api/searchindex.json.gz
Binary file not shown.
Binary file modified docs/content/api/sections.json.gz
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from pathlib import Path

import dagster as dg
from dagster_components import ComponentRegistry, build_component_defs
from dagster_components import ComponentTypeRegistry, build_component_defs
from dagster_components.lib.pipes_subprocess_script_collection import (
PipesSubprocessScriptCollection,
)

defs = build_component_defs(
code_location_root=Path(__file__).parent.parent,
registry=ComponentRegistry(
registry=ComponentTypeRegistry(
{"pipes_subprocess_script_collection": PipesSubprocessScriptCollection}
),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Component as Component,
ComponentGenerateRequest as ComponentGenerateRequest,
ComponentLoadContext as ComponentLoadContext,
ComponentRegistry as ComponentRegistry,
ComponentTypeRegistry as ComponentTypeRegistry,
component_type as component_type,
)
from dagster_components.core.component_defs_builder import (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import click
from pydantic import TypeAdapter

from dagster_components import ComponentRegistry
from dagster_components import ComponentTypeRegistry
from dagster_components.core.deployment import (
CodeLocationProjectContext,
find_enclosing_code_location_root_path,
Expand Down Expand Up @@ -42,7 +42,9 @@ def generate_component_command(

context = CodeLocationProjectContext.from_code_location_path(
find_enclosing_code_location_root_path(Path.cwd()),
ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib),
ComponentTypeRegistry.from_entry_point_discovery(
builtin_component_lib=builtin_component_lib
),
)
if not context.has_component_type(component_type):
click.echo(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import click

from dagster_components.core.component import ComponentMetadata, ComponentRegistry
from dagster_components.core.component import ComponentTypeMetadata, ComponentTypeRegistry
from dagster_components.core.deployment import (
CodeLocationProjectContext,
find_enclosing_code_location_root_path,
Expand Down Expand Up @@ -34,12 +34,14 @@ def list_component_types_command(ctx: click.Context) -> None:

context = CodeLocationProjectContext.from_code_location_path(
find_enclosing_code_location_root_path(Path.cwd()),
ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib),
ComponentTypeRegistry.from_entry_point_discovery(
builtin_component_lib=builtin_component_lib
),
)
output: Dict[str, Any] = {}
for key, component_type in context.list_component_types():
package, name = key.rsplit(".", 1)
output[key] = ComponentMetadata(
output[key] = ComponentTypeMetadata(
name=name,
package=package,
**component_type.get_metadata(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def build_defs(self, context: "ComponentLoadContext") -> Definitions: ...
def load(cls, context: "ComponentLoadContext") -> Self: ...

@classmethod
def get_metadata(cls) -> "ComponentInternalMetadata":
def get_metadata(cls) -> "ComponentTypeInternalMetadata":
docstring = cls.__doc__
clean_docstring = _clean_docstring(docstring) if docstring else None

Expand Down Expand Up @@ -104,14 +104,14 @@ def _get_click_cli_help(command: click.Command) -> str:
return formatter.getvalue()


class ComponentInternalMetadata(TypedDict):
class ComponentTypeInternalMetadata(TypedDict):
summary: Optional[str]
description: Optional[str]
generate_params_schema: Optional[Any] # json schema
component_params_schema: Optional[Any] # json schema


class ComponentMetadata(ComponentInternalMetadata):
class ComponentTypeMetadata(ComponentTypeInternalMetadata):
name: str
package: str

Expand All @@ -129,27 +129,28 @@ def get_entry_points_from_python_environment(group: str) -> Sequence[importlib.m
BUILTIN_TEST_COMPONENT_ENTRY_POINT = ".".join([BUILTIN_COMPONENTS_ENTRY_POINT_BASE, "test"])


class ComponentRegistry:
class ComponentTypeRegistry:
@classmethod
def from_entry_point_discovery(
cls, builtin_component_lib: str = BUILTIN_MAIN_COMPONENT_ENTRY_POINT
) -> "ComponentRegistry":
"""Discover components registered in the Python environment via the `dagster_components` entry point group.
) -> "ComponentTypeRegistry":
"""Discover component types registered in the Python environment via the
`dagster_components` entry point group.

`dagster-components` itself registers multiple component entry points. We call these
"builtin" component libraries. The `dagster_components` entry point resolves to published
components and is loaded by default. Other entry points resolve to various sets of test
components. This method will only ever load one builtin component library.
component types and is loaded by default. Other entry points resolve to various sets of test
component types. This method will only ever load one builtin component library.

Args:
builtin-component-lib (str): Specifies the builtin components library to load. Builtin
copmonents libraries are defined under entry points with names matching the pattern
`dagster_components*`. Only one builtin component library can be loaded at a time.
Defaults to `dagster_components`, the standard set of published components.
builtin-component-lib (str): Specifies the builtin components library to load. Built-in
component libraries are defined under entry points with names matching the pattern
`dagster_components*`. Only one built-in component library can be loaded at a time.
Defaults to `dagster_components`, the standard set of published component types.
"""
components: Dict[str, Type[Component]] = {}
component_types: Dict[str, Type[Component]] = {}
for entry_point in get_entry_points_from_python_environment(COMPONENTS_ENTRY_POINT_GROUP):
# Skip builtin entry points that are not the specified builtin component library.
# Skip built-in entry points that are not the specified builtin component library.
if (
entry_point.name.startswith(BUILTIN_COMPONENTS_ENTRY_POINT_BASE)
and not entry_point.name == builtin_component_lib
Expand All @@ -162,44 +163,44 @@ def from_entry_point_discovery(
f"Invalid entry point {entry_point.name} in group {COMPONENTS_ENTRY_POINT_GROUP}. "
f"Value expected to be a module, got {root_module}."
)
for component in get_registered_components_in_module(root_module):
key = f"{entry_point.name}.{get_component_name(component)}"
components[key] = component
for component_type in get_registered_component_types_in_module(root_module):
key = f"{entry_point.name}.{get_component_type_name(component_type)}"
component_types[key] = component_type

return cls(components)
return cls(component_types)

def __init__(self, components: Dict[str, Type[Component]]):
self._components: Dict[str, Type[Component]] = copy.copy(components)
def __init__(self, component_types: Dict[str, Type[Component]]):
self._component_types: Dict[str, Type[Component]] = copy.copy(component_types)

@staticmethod
def empty() -> "ComponentRegistry":
return ComponentRegistry({})
def empty() -> "ComponentTypeRegistry":
return ComponentTypeRegistry({})

def register(self, name: str, component: Type[Component]) -> None:
if name in self._components:
def register(self, name: str, component_type: Type[Component]) -> None:
if name in self._component_types:
raise DagsterError(f"There is an existing component registered under {name}")
self._components[name] = component
self._component_types[name] = component_type

def has(self, name: str) -> bool:
return name in self._components
return name in self._component_types

def get(self, name: str) -> Type[Component]:
return self._components[name]
return self._component_types[name]

def keys(self) -> Iterable[str]:
return self._components.keys()
return self._component_types.keys()

def __repr__(self) -> str:
return f"<ComponentRegistry {list(self._components.keys())}>"
return f"<ComponentRegistry {list(self._component_types.keys())}>"


def get_registered_components_in_module(module: ModuleType) -> Iterable[Type[Component]]:
def get_registered_component_types_in_module(module: ModuleType) -> Iterable[Type[Component]]:
from dagster._core.definitions.module_loaders.load_assets_from_modules import (
find_subclasses_in_module,
)

for component in find_subclasses_in_module(module, (Component,)):
if is_registered_component(component):
if is_registered_component_type(component):
yield component


Expand All @@ -209,20 +210,20 @@ def get_registered_components_in_module(module: ModuleType) -> Iterable[Type[Com
@dataclass
class ComponentLoadContext:
resources: Mapping[str, object]
registry: ComponentRegistry
registry: ComponentTypeRegistry
decl_node: Optional[ComponentDeclNode]
templated_value_resolver: TemplatedValueResolver

@staticmethod
def for_test(
*,
resources: Optional[Mapping[str, object]] = None,
registry: Optional[ComponentRegistry] = None,
registry: Optional[ComponentTypeRegistry] = None,
decl_node: Optional[ComponentDeclNode] = None,
) -> "ComponentLoadContext":
return ComponentLoadContext(
resources=resources or {},
registry=registry or ComponentRegistry.empty(),
registry=registry or ComponentTypeRegistry.empty(),
decl_node=decl_node,
templated_value_resolver=TemplatedValueResolver.default(),
)
Expand Down Expand Up @@ -293,13 +294,13 @@ def wrapper(actual_cls: Type[Component]) -> Type[Component]:
return cls


def is_registered_component(cls: Type) -> bool:
def is_registered_component_type(cls: Type) -> bool:
return hasattr(cls, COMPONENT_REGISTRY_KEY_ATTR)


def get_component_name(component_type: Type[Component]) -> str:
def get_component_type_name(component_type: Type[Component]) -> str:
check.param_invariant(
is_registered_component(component_type),
is_registered_component_type(component_type),
"component_type",
"Expected a registered component. Use @component to register a component.",
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@
from dagster_components.core.component import (
Component,
ComponentLoadContext,
ComponentRegistry,
ComponentTypeRegistry,
TemplatedValueResolver,
get_component_name,
is_registered_component,
get_component_type_name,
is_registered_component_type,
)
from dagster_components.core.component_decl_builder import (
ComponentFolder,
Expand Down Expand Up @@ -54,7 +54,7 @@ def load_components_from_context(context: ComponentLoadContext) -> Sequence[Comp


def component_type_from_yaml_decl(
registry: ComponentRegistry, decl_node: YamlComponentDecl
registry: ComponentTypeRegistry, decl_node: YamlComponentDecl
) -> Type[Component]:
parsed_defs = decl_node.component_file_model
if parsed_defs.type.startswith("."):
Expand All @@ -69,8 +69,8 @@ def component_type_from_yaml_decl(
for _name, obj in inspect.getmembers(module, inspect.isclass):
assert isinstance(obj, Type)
if (
is_registered_component(obj)
and get_component_name(obj) == component_registry_key
is_registered_component_type(obj)
and get_component_type_name(obj) == component_registry_key
):
return obj

Expand All @@ -91,7 +91,7 @@ def build_components_from_component_folder(

def build_defs_from_component_path(
path: Path,
registry: ComponentRegistry,
registry: ComponentTypeRegistry,
resources: Mapping[str, object],
) -> "Definitions":
"""Build a definitions object from a folder within the components hierarchy."""
Expand Down Expand Up @@ -128,7 +128,7 @@ def defs_from_components(
def build_component_defs(
code_location_root: Path,
resources: Optional[Mapping[str, object]] = None,
registry: Optional["ComponentRegistry"] = None,
registry: Optional["ComponentTypeRegistry"] = None,
components_folder: Optional[Path] = None,
) -> "Definitions":
"""Build a Definitions object for all the component instances in a given code location.
Expand All @@ -141,7 +141,7 @@ def build_component_defs(

context = CodeLocationProjectContext.from_code_location_path(
code_location_root,
registry or ComponentRegistry.from_entry_point_discovery(),
registry or ComponentTypeRegistry.from_entry_point_discovery(),
components_folder=components_folder,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from dagster._core.errors import DagsterError
from typing_extensions import Self

from dagster_components.core.component import Component, ComponentRegistry
from dagster_components.core.component import Component, ComponentTypeRegistry

# Code location
_CODE_LOCATION_CUSTOM_COMPONENTS_DIR: Final = "lib"
Expand Down Expand Up @@ -49,7 +49,7 @@ class CodeLocationProjectContext:
def from_code_location_path(
cls,
path: Path,
component_registry: "ComponentRegistry",
component_registry: "ComponentTypeRegistry",
components_folder: Optional[Path] = None,
) -> Self:
if not _is_code_location_root(path):
Expand All @@ -71,7 +71,7 @@ def __init__(
self,
root_path: str,
name: str,
component_registry: "ComponentRegistry",
component_registry: "ComponentTypeRegistry",
components_folder: Path,
):
self._root_path = root_path
Expand All @@ -88,7 +88,7 @@ def component_types_root_module(self) -> str:
return f"{self._name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}"

@property
def component_registry(self) -> "ComponentRegistry":
def component_registry(self) -> "ComponentTypeRegistry":
return self._component_registry

def has_component_type(self, name: str) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@
from pathlib import Path

from dagster._core.definitions.definitions_class import Definitions
from dagster_components.core.component import ComponentRegistry, get_registered_components_in_module
from dagster_components.core.component import (
ComponentTypeRegistry,
get_registered_component_types_in_module,
)
from dagster_components.core.component_defs_builder import (
build_defs_from_component_path,
get_component_name,
get_component_type_name,
)
from dagster_components.core.deployment import CodeLocationProjectContext

Expand All @@ -24,13 +27,13 @@ def load_test_component_project_context() -> CodeLocationProjectContext:
dc_module = importlib.import_module(package_name)

components = {}
for component in get_registered_components_in_module(dc_module):
key = f"dagster_components.{get_component_name(component)}"
for component in get_registered_component_types_in_module(dc_module):
key = f"dagster_components.{get_component_type_name(component)}"
components[key] = component

return CodeLocationProjectContext(
root_path=str(Path(__file__).parent),
name="test",
component_registry=ComponentRegistry(components),
component_registry=ComponentTypeRegistry(components),
components_folder=Path(__file__).parent / "components",
)
Loading
Loading