From f73593ef51d066e719b4c0b0a5fdf2dd04f0c136 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Tue, 26 Nov 2024 16:24:25 -0500 Subject: [PATCH] [components] Initial implementation of component project scaffolding (#26104) ## Summary & Motivation This is an initial implementation of scaffolding for components. It adds a new CLI `dg` rather than a subcommand to `dagster`. There is other work in flight but this seemed like a good place to cut the bottom of the stack. Implements: - `dg generate deployment` - `dg generate code_location` - `dg generate component-type` - `dg generate component-instance` What is NOT in this PR: - global component registry based on entrypoints-- instead this only resolves component types local to the code location - anything resembling final `Component` API. This uses a very bare-bones prototype implementing `build_defs` and `generate_files` that will be replaced. - Calling out to `uv` in scaffolding commands. - Thoroughly polished output for scaffold commands. - Final names for commands (`component-type`, `component-instance` are questionable) All of ^^ will come upstack or from @OwenKephart efforts. Note that it was necessary to expose `Component` from top-level `dagster` so that realistic generated files could be used (`from dagster import Component`). To avoid exposing this to users I added it as a dynamic import. Note also there is some weirdness with `sys.path`, which by default has `""` as first entry (allowing imports from cwd) but which for some reason is missing this entry in the test environment. I've hacked it in with some code that will probably go away once the underyling issue can be identified. ## How I Tested These Changes New unit tests. --- .gitignore | 2 - python_modules/dagster/dagster/__init__.py | 10 + .../dagster/dagster/_cli/__init__.py | 2 +- .../dagster/dagster/_components/__init__.py | 204 ++++++++++++++++++ .../dagster/_components/cli/__init__.py | 28 +++ .../dagster/_components/cli/generate.py | 102 +++++++++ .../definitions/load_assets_from_modules.py | 20 +- .../dagster/dagster/_generate/generate.py | 58 ++++- .../__init__.py | 0 .../components/.gitkeep | 0 .../lib/__init__.py | 0 .../__init__.py | 0 .../pyproject.toml.jinja | 24 +++ .../COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja | 10 + .../workflows/dagster-cloud-deploy.yaml.jinja | 0 .../code_locations/.gitkeep | 0 .../dagster_cloud.yaml.jinja | 0 .../dagster/dagster/_utils/__init__.py | 8 + .../cli_tests/test_generate_commands.py | 161 ++++++++++++++ .../components_tests/__init__.py | 0 python_modules/dagster/setup.py | 1 + 21 files changed, 621 insertions(+), 9 deletions(-) create mode 100644 python_modules/dagster/dagster/_components/__init__.py create mode 100644 python_modules/dagster/dagster/_components/cli/__init__.py create mode 100644 python_modules/dagster/dagster/_components/cli/generate.py create mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py create mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/components/.gitkeep create mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/lib/__init__.py create mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py create mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja create mode 100644 python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja create mode 100644 python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/.github/workflows/dagster-cloud-deploy.yaml.jinja create mode 100644 python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/code_locations/.gitkeep create mode 100644 python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/dagster_cloud.yaml.jinja create mode 100644 python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py create mode 100644 python_modules/dagster/dagster_tests/components_tests/__init__.py diff --git a/.gitignore b/.gitignore index 680c845a06756..b73feee7312a9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ parts/ sdist/ var/ diff --git a/python_modules/dagster/dagster/__init__.py b/python_modules/dagster/dagster/__init__.py index 2e3e391b6d063..bfaaf2f04a238 100644 --- a/python_modules/dagster/dagster/__init__.py +++ b/python_modules/dagster/dagster/__init__.py @@ -670,6 +670,13 @@ } +# Use this to expose symbols from top-level dagster for testing purposes during development, before +# we want to expose them to users. +_HIDDEN: Final[Mapping[str, str]] = { + "Component": "dagster._components", +} + + def __getattr__(name: str) -> TypingAny: if name in _DEPRECATED: module, breaking_version, additional_warn_text = _DEPRECATED[name] @@ -687,6 +694,9 @@ def __getattr__(name: str) -> TypingAny: stacklevel=stacklevel, ) return value + elif name in _HIDDEN: + module = _HIDDEN[name] + return getattr(importlib.import_module(module), name) else: raise AttributeError(f"module '{__name__}' has no attribute '{name}'") diff --git a/python_modules/dagster/dagster/_cli/__init__.py b/python_modules/dagster/dagster/_cli/__init__.py index 62d7c5fda109d..5e1e1e72066f4 100644 --- a/python_modules/dagster/dagster/_cli/__init__.py +++ b/python_modules/dagster/dagster/_cli/__init__.py @@ -47,4 +47,4 @@ def group(): def main(): - cli(auto_envvar_prefix=ENV_PREFIX) # pylint:disable=E1123 + cli(auto_envvar_prefix=ENV_PREFIX) diff --git a/python_modules/dagster/dagster/_components/__init__.py b/python_modules/dagster/dagster/_components/__init__.py new file mode 100644 index 0000000000000..bd1ff9665bbc6 --- /dev/null +++ b/python_modules/dagster/dagster/_components/__init__.py @@ -0,0 +1,204 @@ +import importlib.util +import os +import sys +from abc import ABC, abstractmethod +from types import ModuleType +from typing import TYPE_CHECKING, ClassVar, Dict, Final, Iterable, Optional, Type + +from typing_extensions import Self + +from dagster._core.errors import DagsterError +from dagster._utils import snakecase + +if TYPE_CHECKING: + from dagster._core.definitions.definitions_class import Definitions + + +class Component(ABC): + name: ClassVar[Optional[str]] = None + + @classmethod + def registered_name(cls): + return cls.name or snakecase(cls.__name__) + + @classmethod + def generate_files(cls) -> None: + raise NotImplementedError() + + @abstractmethod + def build_defs(self) -> "Definitions": ... + + +def is_inside_deployment_project(path: str = ".") -> bool: + try: + _resolve_deployment_root_path(path) + return True + except DagsterError: + return False + + +def _resolve_deployment_root_path(path: str) -> str: + current_path = os.path.abspath(path) + while not _is_deployment_root(current_path): + current_path = os.path.dirname(current_path) + if current_path == "/": + raise DagsterError("Cannot find deployment root") + return current_path + + +def is_inside_code_location_project(path: str = ".") -> bool: + try: + _resolve_code_location_root_path(path) + return True + except DagsterError: + return False + + +def _resolve_code_location_root_path(path: str) -> str: + current_path = os.path.abspath(path) + while not _is_code_location_root(current_path): + current_path = os.path.dirname(current_path) + if current_path == "/": + raise DagsterError("Cannot find code location root") + return current_path + + +def _is_deployment_root(path: str) -> bool: + return os.path.exists(os.path.join(path, "code_locations")) + + +def _is_code_location_root(path: str) -> bool: + return os.path.basename(os.path.dirname(path)) == "code_locations" + + +# Deployment +_DEPLOYMENT_CODE_LOCATIONS_DIR: Final = "code_locations" + +# Code location +_CODE_LOCATION_CUSTOM_COMPONENTS_DIR: Final = "lib" +_CODE_LOCATION_COMPONENT_INSTANCES_DIR: Final = "components" + + +class DeploymentProjectContext: + @classmethod + def from_path(cls, path: str) -> Self: + return cls(root_path=_resolve_deployment_root_path(path)) + + def __init__(self, root_path: str): + self._root_path = root_path + + @property + def deployment_root(self) -> str: + return self._root_path + + @property + def code_location_root_path(self) -> str: + return os.path.join(self._root_path, _DEPLOYMENT_CODE_LOCATIONS_DIR) + + def has_code_location(self, name: str) -> bool: + return os.path.exists(os.path.join(self._root_path, "code_locations", name)) + + +class CodeLocationProjectContext: + @classmethod + def from_path(cls, path: str) -> Self: + root_path = _resolve_code_location_root_path(path) + name = os.path.basename(root_path) + component_registry = ComponentRegistry() + + # TODO: Rm when a more robust solution is implemented + # Make sure we can import from the cwd + if sys.path[0] != "": + sys.path.insert(0, "") + + components_lib_module = f"{name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}" + module = importlib.import_module(components_lib_module) + register_components_in_module(component_registry, module) + + return cls( + deployment_context=DeploymentProjectContext.from_path(path), + root_path=root_path, + name=os.path.basename(root_path), + component_registry=component_registry, + ) + + def __init__( + self, + deployment_context: DeploymentProjectContext, + root_path: str, + name: str, + component_registry: "ComponentRegistry", + ): + self._deployment_context = deployment_context + self._root_path = root_path + self._name = name + self._component_registry = component_registry + + @property + def deployment_context(self) -> DeploymentProjectContext: + return self._deployment_context + + @property + def component_types_root_path(self) -> str: + return os.path.join(self._root_path, self._name, _CODE_LOCATION_CUSTOM_COMPONENTS_DIR) + + @property + def component_types_root_module(self) -> str: + return f"{self._name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}" + + def has_component_type(self, name: str) -> bool: + return self._component_registry.has(name) + + def get_component_type(self, name: str) -> Type[Component]: + if not self.has_component_type(name): + raise DagsterError(f"No component type named {name}") + return self._component_registry.get(name) + + @property + def component_instances_root_path(self) -> str: + return os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR) + + @property + def component_instances(self) -> Iterable[str]: + return os.listdir( + os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR) + ) + + def has_component_instance(self, name: str) -> bool: + return os.path.exists( + os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR, name) + ) + + +class ComponentRegistry: + def __init__(self): + self._components: Dict[str, Type[Component]] = {} + + def register(self, name: str, component: Type[Component]) -> None: + self._components[name] = component + + def has(self, name: str) -> bool: + return name in self._components + + def get(self, name: str) -> Type[Component]: + return self._components[name] + + def keys(self) -> Iterable[str]: + return self._components.keys() + + def __repr__(self): + return f"" + + +def register_components_in_module(registry: ComponentRegistry, root_module: ModuleType) -> None: + from dagster._core.definitions.load_assets_from_modules import ( + find_modules_in_package, + find_subclasses_in_module, + ) + + for module in find_modules_in_package(root_module): + for component in find_subclasses_in_module(module, (Component,)): + if component is Component: + continue + name = f"{module.__name__}[{component.registered_name()}]" + registry.register(name, component) diff --git a/python_modules/dagster/dagster/_components/cli/__init__.py b/python_modules/dagster/dagster/_components/cli/__init__.py new file mode 100644 index 0000000000000..34fdd17211667 --- /dev/null +++ b/python_modules/dagster/dagster/_components/cli/__init__.py @@ -0,0 +1,28 @@ +import click + +from dagster._components.cli.generate import generate_cli +from dagster.version import __version__ + + +def create_dagster_components_cli(): + commands = { + "generate": generate_cli, + } + + @click.group( + commands=commands, + context_settings={"max_content_width": 120, "help_option_names": ["-h", "--help"]}, + ) + @click.version_option(__version__, "--version", "-v") + def group(): + """CLI tools for working with Dagster.""" + + return group + + +ENV_PREFIX = "DG_CLI" +cli = create_dagster_components_cli() + + +def main(): + cli(auto_envvar_prefix=ENV_PREFIX) diff --git a/python_modules/dagster/dagster/_components/cli/generate.py b/python_modules/dagster/dagster/_components/cli/generate.py new file mode 100644 index 0000000000000..49c7d9a651e09 --- /dev/null +++ b/python_modules/dagster/dagster/_components/cli/generate.py @@ -0,0 +1,102 @@ +import os +import sys + +import click + +from dagster._components import ( + CodeLocationProjectContext, + DeploymentProjectContext, + is_inside_code_location_project, + is_inside_deployment_project, +) +from dagster._generate.generate import ( + generate_code_location, + generate_component_instance, + generate_component_type, + generate_deployment, +) + + +@click.group(name="generate") +def generate_cli(): + """Commands for generating Dagster components and related entities.""" + + +@click.command(name="deployment") +@click.argument("path", type=str) +def generate_deployment_command(path: str) -> None: + """Generate a Dagster deployment instance.""" + dir_abspath = os.path.abspath(path) + if os.path.exists(dir_abspath): + click.echo( + click.style(f"A file or directory at {dir_abspath} already exists. ", fg="red") + + "\nPlease delete the contents of this path or choose another location." + ) + sys.exit(1) + generate_deployment(path) + + +@click.command(name="code-location") +@click.argument("name", type=str) +def generate_code_location_command(name: str) -> None: + """Generate a Dagster code location inside a component.""" + if not is_inside_deployment_project(): + click.echo( + click.style("This command must be run inside a Dagster deployment project.", fg="red") + ) + sys.exit(1) + + context = DeploymentProjectContext.from_path(os.getcwd()) + if context.has_code_location(name): + click.echo(click.style(f"A code location named {name} already exists.", fg="red")) + sys.exit(1) + + code_location_path = os.path.join(context.code_location_root_path, name) + generate_code_location(code_location_path) + + +@click.command(name="component-type") +@click.argument("name", type=str) +def generate_component_type_command(name: str) -> None: + """Generate a Dagster component instance.""" + if not is_inside_code_location_project(): + click.echo( + click.style( + "This command must be run inside a Dagster code location project.", fg="red" + ) + ) + sys.exit(1) + + context = CodeLocationProjectContext.from_path(os.getcwd()) + if context.has_component_type(name): + click.echo(click.style(f"A component type named `{name}` already exists.", fg="red")) + sys.exit(1) + + generate_component_type(context.component_types_root_path, name) + + +@click.command(name="component-instance") +@click.argument("component-type", type=str) +@click.argument("name", type=str) +def generate_component_instance_command(component_type: str, name: str) -> None: + """Generate a Dagster component instance.""" + if not is_inside_code_location_project(): + click.echo( + click.style( + "This command must be run inside a Dagster code location project.", fg="red" + ) + ) + sys.exit(1) + + context = CodeLocationProjectContext.from_path(os.getcwd()) + if not context.has_component_type(component_type): + click.echo( + click.style(f"No component type `{component_type}` could be resolved.", fg="red") + ) + sys.exit(1) + elif context.has_component_instance(name): + click.echo(click.style(f"A component instance named `{name}` already exists.", fg="red")) + sys.exit(1) + + component_type_cls = context.get_component_type(component_type) + generate_component_instance(context.component_instances_root_path, name, component_type_cls) diff --git a/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py b/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py index 4291f39488730..284d76795e505 100644 --- a/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py +++ b/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py @@ -2,7 +2,7 @@ import pkgutil from importlib import import_module from types import ModuleType -from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, Union, cast +from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, Type, Union, cast import dagster._check as check from dagster._core.definitions.asset_key import ( @@ -23,8 +23,11 @@ from dagster._core.errors import DagsterInvalidDefinitionError -def find_objects_in_module_of_types(module: ModuleType, types) -> Iterator: - """Yields objects of the given type(s).""" +def find_objects_in_module_of_types( + module: ModuleType, + types: Union[Type, Tuple[Type, ...]], +) -> Iterator: + """Yields instances or subclasses of the given type(s).""" for attr in dir(module): value = getattr(module, attr) if isinstance(value, types): @@ -33,6 +36,17 @@ def find_objects_in_module_of_types(module: ModuleType, types) -> Iterator: yield from value +def find_subclasses_in_module( + module: ModuleType, + types: Union[Type, Tuple[Type, ...]], +) -> Iterator: + """Yields instances or subclasses of the given type(s).""" + for attr in dir(module): + value = getattr(module, attr) + if isinstance(value, type) and issubclass(value, types): + yield value + + def assets_from_modules( modules: Iterable[ModuleType], extra_source_assets: Optional[Sequence[SourceAsset]] = None ) -> Tuple[Sequence[AssetsDefinition], Sequence[SourceAsset], Sequence[CacheableAssetsDefinition]]: diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index a3ce06873ea1b..9d84513afa691 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,10 +1,12 @@ import os import posixpath -from typing import List, Optional +from typing import Any, List, Optional, Type import click import jinja2 +from dagster._components import Component +from dagster._utils import camelcase, pushd from dagster.version import __version__ as dagster_version DEFAULT_EXCLUDES: List[str] = [ @@ -14,11 +16,57 @@ ".DS_Store", ".ruff_cache", "tox.ini", + ".gitkeep", # dummy file that allows empty directories to be checked into git ] PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" +def generate_deployment(path: str) -> None: + click.echo(f"Creating a Dagster deployment at {path}.") + + generate_project( + path=path, + name_placeholder="DEPLOYMENT_NAME_PLACEHOLDER", + templates_path=os.path.join( + os.path.dirname(__file__), "templates", "DEPLOYMENT_NAME_PLACEHOLDER" + ), + ) + + +def generate_code_location(path: str) -> None: + click.echo(f"Creating a Dagster code location at {path}.") + + generate_project( + path=path, + name_placeholder="CODE_LOCATION_NAME_PLACEHOLDER", + templates_path=os.path.join( + os.path.dirname(__file__), "templates", "CODE_LOCATION_NAME_PLACEHOLDER" + ), + ) + + +def generate_component_type(root_path: str, name: str) -> None: + click.echo(f"Creating a Dagster component type at {root_path}/{name}.py.") + + generate_project( + path=root_path, + name_placeholder="COMPONENT_TYPE_NAME_PLACEHOLDER", + templates_path=os.path.join(os.path.dirname(__file__), "templates", "COMPONENT_TYPE"), + project_name=name, + component_type_class_name=camelcase(name), + ) + + +def generate_component_instance(root_path: str, name: str, component_type: Type[Component]) -> None: + click.echo(f"Creating a Dagster component instance at {root_path}/{name}.py.") + + component_instance_root_path = os.path.join(root_path, name) + os.mkdir(component_instance_root_path) + with pushd(component_instance_root_path): + component_type.generate_files() + + def generate_repository(path: str): REPO_NAME_PLACEHOLDER = "REPO_NAME_PLACEHOLDER" @@ -40,6 +88,8 @@ def generate_project( excludes: Optional[List[str]] = None, name_placeholder: str = PROJECT_NAME_PLACEHOLDER, templates_path: str = PROJECT_NAME_PLACEHOLDER, + project_name: Optional[str] = None, + **other_template_vars: Any, ): """Renders templates for Dagster project.""" excludes = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes @@ -47,8 +97,9 @@ def generate_project( click.echo(f"Creating a Dagster project at {path}.") normalized_path = os.path.normpath(path) - project_name: str = os.path.basename(normalized_path).replace("-", "_") - os.mkdir(normalized_path) + project_name = project_name or os.path.basename(normalized_path).replace("-", "_") + if not os.path.exists(normalized_path): + os.mkdir(normalized_path) project_template_path: str = os.path.join( os.path.dirname(__file__), "templates", templates_path @@ -103,6 +154,7 @@ def generate_project( code_location_name=project_name, dagster_version=dagster_version, project_name=project_name, + **other_template_vars, ) ) f.write("\n") diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/components/.gitkeep b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/components/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/lib/__init__.py b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/lib/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja new file mode 100644 index 0000000000000..2545276208832 --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja @@ -0,0 +1,24 @@ +[project] +name = "{{ project_name }}" +requires-python = ">=3.9,<3.13" +dependencies = [ + "dagster", + "dagster-cloud", +] + +[project.optional-dependencies] +dev = [ + "dagster-webserver", + "pytest", +] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "{{ project_name }}.definitions" +project_name = "{{ project_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ project_name }}_tests"] diff --git a/python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja b/python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja new file mode 100644 index 0000000000000..960cb28c5d708 --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja @@ -0,0 +1,10 @@ +from dagster import Component, Definitions + +class {{ component_type_class_name }}(Component): + + @classmethod + def generate_files(cls) -> None: + ... + + def build_defs(self) -> Definitions: + ... diff --git a/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/.github/workflows/dagster-cloud-deploy.yaml.jinja b/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/.github/workflows/dagster-cloud-deploy.yaml.jinja new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/code_locations/.gitkeep b/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/code_locations/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/dagster_cloud.yaml.jinja b/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/dagster_cloud.yaml.jinja new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_utils/__init__.py b/python_modules/dagster/dagster/_utils/__init__.py index 575796a7d9309..fce1525d93856 100644 --- a/python_modules/dagster/dagster/_utils/__init__.py +++ b/python_modules/dagster/dagster/_utils/__init__.py @@ -201,6 +201,14 @@ def camelcase(string: str) -> str: ) +def snakecase(string: str) -> str: + # Add an underscore before capital letters and lower the case + string = re.sub(r"(? Tuple[T, U]: check.mapping_param(ddict, "ddict") check.param_invariant(len(ddict) == 1, "ddict", "Expected dict with single item") diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py new file mode 100644 index 0000000000000..82e0504b52cdd --- /dev/null +++ b/python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py @@ -0,0 +1,161 @@ +import importlib +import os +import sys +from contextlib import contextmanager +from pathlib import Path + +from click.testing import CliRunner +from dagster._components import CodeLocationProjectContext +from dagster._components.cli.generate import ( + generate_code_location_command, + generate_component_instance_command, + generate_component_type_command, + generate_deployment_command, +) + + +def _ensure_cwd_on_sys_path(): + if sys.path[0] != "": + sys.path.insert(0, "") + + +def _assert_module_imports(module_name: str): + _ensure_cwd_on_sys_path() + assert importlib.import_module(module_name) + + +@contextmanager +def clean_module_cache(module_name: str): + prefix = f"{module_name}." + keys_to_del = { + key for key in sys.modules.keys() if key == module_name or key.startswith(prefix) + } + for key in keys_to_del: + del sys.modules[key] + yield + + +def test_generate_deployment_command_success(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(generate_deployment_command, ["foo"]) + assert result.exit_code == 0 + assert Path("foo").exists() + assert Path("foo/.github").exists() + assert Path("foo/.github/workflows").exists() + assert Path("foo/.github/workflows/dagster-cloud-deploy.yaml").exists() + assert Path("foo/dagster_cloud.yaml").exists() + assert Path("foo/code_locations").exists() + + +def test_generate_deployment_command_path_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + os.mkdir("foo") + result = runner.invoke(generate_deployment_command, ["foo"]) + assert result.exit_code != 0 + assert "already exists" in result.output + + +def test_generate_code_location_success(): + runner = CliRunner() + with runner.isolated_filesystem(): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code == 0 + assert Path("code_locations/bar").exists() + assert Path("code_locations/bar/bar").exists() + assert Path("code_locations/bar/bar/lib").exists() + assert Path("code_locations/bar/bar/components").exists() + assert Path("code_locations/bar/bar_tests").exists() + assert Path("code_locations/bar/pyproject.toml").exists() + + +def test_generate_code_location_outside_deployment_fails(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code != 0 + assert "must be run inside a Dagster deployment project" in result.output + + +def test_generate_code_location_already_exists_fails(): + runner = CliRunner() + with runner.isolated_filesystem(): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code == 0 + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code != 0 + assert "already exists" in result.output + + +def test_generate_component_type_success(): + runner = CliRunner() + with runner.isolated_filesystem(), clean_module_cache("bar"): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + # set up code location + runner.invoke(generate_code_location_command, ["bar"]) + os.chdir("code_locations/bar") + + result = runner.invoke(generate_component_type_command, ["baz"]) + assert result.exit_code == 0 + assert Path("bar/lib/baz.py").exists() + _assert_module_imports("bar.lib.baz") + context = CodeLocationProjectContext.from_path(os.getcwd()) + assert context.has_component_type("bar.lib.baz[baz]") + + +_SAMPLE_COMPONENT_TYPE = """ +from dagster import Component, Definitions, PipesSubprocessClient + +_SAMPLE_PIPES_SCRIPT = \""" +from dagster_pipes import open_dagster_pipes + +context = open_dagster_pipes() +context.report_asset_materialization({"alpha": "beta"}) +\""" + +class Baz(Component): + + @classmethod + def generate_files(cls): + with open("sample.py", "w") as f: + f.write(_SAMPLE_PIPES_SCRIPT) + + def build_defs(self) -> Definitions: + @asset + def foo(): + PipesSubprocessClient("foo.py").run(context, command=["python", "sample.py"]) + + return Definitions( + assets=[foo] + ) + +""" + + +def test_generate_component_instance_success(): + runner = CliRunner() + _ensure_cwd_on_sys_path() + with runner.isolated_filesystem(), clean_module_cache("bar"): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + # set up code location + runner.invoke(generate_code_location_command, ["bar"]) + os.chdir("code_locations/bar") + # set up component type + with open("bar/lib/baz.py", "w") as f: + f.write(_SAMPLE_COMPONENT_TYPE) + result = runner.invoke(generate_component_instance_command, ["bar.lib.baz[baz]", "qux"]) + assert result.exit_code == 0 + assert Path("bar/components/qux").exists() diff --git a/python_modules/dagster/dagster_tests/components_tests/__init__.py b/python_modules/dagster/dagster_tests/components_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/setup.py b/python_modules/dagster/setup.py index 805126e175aca..2c79ba826eff2 100644 --- a/python_modules/dagster/setup.py +++ b/python_modules/dagster/setup.py @@ -167,6 +167,7 @@ def get_version() -> str: "console_scripts": [ "dagster = dagster.cli:main", "dagster-daemon = dagster.daemon.cli:main", + "dg = dagster._components.cli:main", ] }, )