Skip to content

Commit

Permalink
[components] ComponentRegistry --> ComponentTypeRegistry
Browse files Browse the repository at this point in the history
  • Loading branch information
schrockn committed Dec 24, 2024
1 parent ee30a4a commit ae19563
Show file tree
Hide file tree
Showing 12 changed files with 44 additions and 36 deletions.
Binary file modified docs/content/api/modules.json.gz
Binary file not shown.
Binary file modified docs/content/api/searchindex.json.gz
Binary file not shown.
Binary file modified docs/content/api/sections.json.gz
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from pathlib import Path

import dagster as dg
from dagster_components import ComponentRegistry, build_component_defs
from dagster_components import ComponentTypeRegistry, build_component_defs
from dagster_components.lib.pipes_subprocess_script_collection import (
PipesSubprocessScriptCollection,
)

defs = build_component_defs(
code_location_root=Path(__file__).parent.parent,
registry=ComponentRegistry(
registry=ComponentTypeRegistry(
{"pipes_subprocess_script_collection": PipesSubprocessScriptCollection}
),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Component as Component,
ComponentGenerateRequest as ComponentGenerateRequest,
ComponentLoadContext as ComponentLoadContext,
ComponentRegistry as ComponentRegistry,
ComponentTypeRegistry as ComponentTypeRegistry,
component_type as component_type,
)
from dagster_components.core.component_defs_builder import (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import click
from pydantic import TypeAdapter

from dagster_components import ComponentRegistry
from dagster_components import ComponentTypeRegistry
from dagster_components.core.deployment import (
CodeLocationProjectContext,
find_enclosing_code_location_root_path,
Expand Down Expand Up @@ -42,7 +42,9 @@ def generate_component_command(

context = CodeLocationProjectContext.from_code_location_path(
find_enclosing_code_location_root_path(Path.cwd()),
ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib),
ComponentTypeRegistry.from_entry_point_discovery(
builtin_component_lib=builtin_component_lib
),
)
if not context.has_component_type(component_type):
click.echo(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import click

from dagster_components.core.component import ComponentMetadata, ComponentRegistry
from dagster_components.core.component import ComponentMetadata, ComponentTypeRegistry
from dagster_components.core.deployment import (
CodeLocationProjectContext,
find_enclosing_code_location_root_path,
Expand Down Expand Up @@ -34,7 +34,9 @@ def list_component_types_command(ctx: click.Context) -> None:

context = CodeLocationProjectContext.from_code_location_path(
find_enclosing_code_location_root_path(Path.cwd()),
ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib),
ComponentTypeRegistry.from_entry_point_discovery(
builtin_component_lib=builtin_component_lib
),
)
output: Dict[str, Any] = {}
for key, component_type in context.list_component_types():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,27 +129,28 @@ def get_entry_points_from_python_environment(group: str) -> Sequence[importlib.m
BUILTIN_TEST_COMPONENT_ENTRY_POINT = ".".join([BUILTIN_COMPONENTS_ENTRY_POINT_BASE, "test"])


class ComponentRegistry:
class ComponentTypeRegistry:
@classmethod
def from_entry_point_discovery(
cls, builtin_component_lib: str = BUILTIN_MAIN_COMPONENT_ENTRY_POINT
) -> "ComponentRegistry":
"""Discover components registered in the Python environment via the `dagster_components` entry point group.
) -> "ComponentTypeRegistry":
"""Discover component types registered in the Python environment via the
`dagster_components` entry point group.
`dagster-components` itself registers multiple component entry points. We call these
"builtin" component libraries. The `dagster_components` entry point resolves to published
components and is loaded by default. Other entry points resolve to various sets of test
components. This method will only ever load one builtin component library.
component types and is loaded by default. Other entry points resolve to various sets of test
component types. This method will only ever load one builtin component library.
Args:
builtin-component-lib (str): Specifies the builtin components library to load. Builtin
copmonents libraries are defined under entry points with names matching the pattern
`dagster_components*`. Only one builtin component library can be loaded at a time.
Defaults to `dagster_components`, the standard set of published components.
builtin-component-lib (str): Specifies the builtin components library to load. Built-in
component libraries are defined under entry points with names matching the pattern
`dagster_components*`. Only one built-in component library can be loaded at a time.
Defaults to `dagster_components`, the standard set of published component types.
"""
components: Dict[str, Type[Component]] = {}
for entry_point in get_entry_points_from_python_environment(COMPONENTS_ENTRY_POINT_GROUP):
# Skip builtin entry points that are not the specified builtin component library.
# Skip built-in entry points that are not the specified builtin component library.
if (
entry_point.name.startswith(BUILTIN_COMPONENTS_ENTRY_POINT_BASE)
and not entry_point.name == builtin_component_lib
Expand All @@ -172,8 +173,8 @@ def __init__(self, components: Dict[str, Type[Component]]):
self._components: Dict[str, Type[Component]] = copy.copy(components)

@staticmethod
def empty() -> "ComponentRegistry":
return ComponentRegistry({})
def empty() -> "ComponentTypeRegistry":
return ComponentTypeRegistry({})

def register(self, name: str, component: Type[Component]) -> None:
if name in self._components:
Expand Down Expand Up @@ -209,20 +210,20 @@ def get_registered_components_in_module(module: ModuleType) -> Iterable[Type[Com
@dataclass
class ComponentLoadContext:
resources: Mapping[str, object]
registry: ComponentRegistry
registry: ComponentTypeRegistry
decl_node: Optional[ComponentDeclNode]
templated_value_resolver: TemplatedValueResolver

@staticmethod
def for_test(
*,
resources: Optional[Mapping[str, object]] = None,
registry: Optional[ComponentRegistry] = None,
registry: Optional[ComponentTypeRegistry] = None,
decl_node: Optional[ComponentDeclNode] = None,
) -> "ComponentLoadContext":
return ComponentLoadContext(
resources=resources or {},
registry=registry or ComponentRegistry.empty(),
registry=registry or ComponentTypeRegistry.empty(),
decl_node=decl_node,
templated_value_resolver=TemplatedValueResolver.default(),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from dagster_components.core.component import (
Component,
ComponentLoadContext,
ComponentRegistry,
ComponentTypeRegistry,
TemplatedValueResolver,
get_component_name,
is_registered_component,
Expand Down Expand Up @@ -54,7 +54,7 @@ def load_components_from_context(context: ComponentLoadContext) -> Sequence[Comp


def component_type_from_yaml_decl(
registry: ComponentRegistry, decl_node: YamlComponentDecl
registry: ComponentTypeRegistry, decl_node: YamlComponentDecl
) -> Type[Component]:
parsed_defs = decl_node.component_file_model
if parsed_defs.type.startswith("."):
Expand Down Expand Up @@ -91,7 +91,7 @@ def build_components_from_component_folder(

def build_defs_from_component_path(
path: Path,
registry: ComponentRegistry,
registry: ComponentTypeRegistry,
resources: Mapping[str, object],
) -> "Definitions":
"""Build a definitions object from a folder within the components hierarchy."""
Expand Down Expand Up @@ -128,7 +128,7 @@ def defs_from_components(
def build_component_defs(
code_location_root: Path,
resources: Optional[Mapping[str, object]] = None,
registry: Optional["ComponentRegistry"] = None,
registry: Optional["ComponentTypeRegistry"] = None,
components_folder: Optional[Path] = None,
) -> "Definitions":
"""Build a Definitions object for all the component instances in a given code location.
Expand All @@ -141,7 +141,7 @@ def build_component_defs(

context = CodeLocationProjectContext.from_code_location_path(
code_location_root,
registry or ComponentRegistry.from_entry_point_discovery(),
registry or ComponentTypeRegistry.from_entry_point_discovery(),
components_folder=components_folder,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from dagster._core.errors import DagsterError
from typing_extensions import Self

from dagster_components.core.component import Component, ComponentRegistry
from dagster_components.core.component import Component, ComponentTypeRegistry

# Code location
_CODE_LOCATION_CUSTOM_COMPONENTS_DIR: Final = "lib"
Expand Down Expand Up @@ -49,7 +49,7 @@ class CodeLocationProjectContext:
def from_code_location_path(
cls,
path: Path,
component_registry: "ComponentRegistry",
component_registry: "ComponentTypeRegistry",
components_folder: Optional[Path] = None,
) -> Self:
if not _is_code_location_root(path):
Expand All @@ -71,7 +71,7 @@ def __init__(
self,
root_path: str,
name: str,
component_registry: "ComponentRegistry",
component_registry: "ComponentTypeRegistry",
components_folder: Path,
):
self._root_path = root_path
Expand All @@ -88,7 +88,7 @@ def component_types_root_module(self) -> str:
return f"{self._name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}"

@property
def component_registry(self) -> "ComponentRegistry":
def component_registry(self) -> "ComponentTypeRegistry":
return self._component_registry

def has_component_type(self, name: str) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@
from pathlib import Path

from dagster._core.definitions.definitions_class import Definitions
from dagster_components.core.component import ComponentRegistry, get_registered_components_in_module
from dagster_components.core.component import (
ComponentTypeRegistry,
get_registered_components_in_module,
)
from dagster_components.core.component_defs_builder import (
build_defs_from_component_path,
get_component_name,
Expand Down Expand Up @@ -31,6 +34,6 @@ def load_test_component_project_context() -> CodeLocationProjectContext:
return CodeLocationProjectContext(
root_path=str(Path(__file__).parent),
name="test",
component_registry=ComponentRegistry(components),
component_registry=ComponentTypeRegistry(components),
components_folder=Path(__file__).parent / "components",
)
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@
Component,
ComponentDeclNode,
ComponentLoadContext,
ComponentRegistry,
ComponentTypeRegistry,
)


def registry() -> ComponentRegistry:
return ComponentRegistry.from_entry_point_discovery()
def registry() -> ComponentTypeRegistry:
return ComponentTypeRegistry.from_entry_point_discovery()


def script_load_context(decl_node: Optional[ComponentDeclNode] = None) -> ComponentLoadContext:
Expand Down

0 comments on commit ae19563

Please sign in to comment.