diff --git a/.gitignore b/.gitignore index 680c845a06756..b73feee7312a9 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ -lib64/ parts/ sdist/ var/ diff --git a/python_modules/dagster/dagster/__init__.py b/python_modules/dagster/dagster/__init__.py index 2e3e391b6d063..bfaaf2f04a238 100644 --- a/python_modules/dagster/dagster/__init__.py +++ b/python_modules/dagster/dagster/__init__.py @@ -670,6 +670,13 @@ } +# Use this to expose symbols from top-level dagster for testing purposes during development, before +# we want to expose them to users. +_HIDDEN: Final[Mapping[str, str]] = { + "Component": "dagster._components", +} + + def __getattr__(name: str) -> TypingAny: if name in _DEPRECATED: module, breaking_version, additional_warn_text = _DEPRECATED[name] @@ -687,6 +694,9 @@ def __getattr__(name: str) -> TypingAny: stacklevel=stacklevel, ) return value + elif name in _HIDDEN: + module = _HIDDEN[name] + return getattr(importlib.import_module(module), name) else: raise AttributeError(f"module '{__name__}' has no attribute '{name}'") diff --git a/python_modules/dagster/dagster/_cli/__init__.py b/python_modules/dagster/dagster/_cli/__init__.py index 62d7c5fda109d..5e1e1e72066f4 100644 --- a/python_modules/dagster/dagster/_cli/__init__.py +++ b/python_modules/dagster/dagster/_cli/__init__.py @@ -47,4 +47,4 @@ def group(): def main(): - cli(auto_envvar_prefix=ENV_PREFIX) # pylint:disable=E1123 + cli(auto_envvar_prefix=ENV_PREFIX) diff --git a/python_modules/dagster/dagster/_components/__init__.py b/python_modules/dagster/dagster/_components/__init__.py new file mode 100644 index 0000000000000..bd1ff9665bbc6 --- /dev/null +++ b/python_modules/dagster/dagster/_components/__init__.py @@ -0,0 +1,204 @@ +import importlib.util +import os +import sys +from abc import ABC, abstractmethod +from types import ModuleType +from typing import TYPE_CHECKING, ClassVar, Dict, Final, Iterable, Optional, Type + +from typing_extensions import Self + +from dagster._core.errors import DagsterError +from dagster._utils import snakecase + +if TYPE_CHECKING: + from dagster._core.definitions.definitions_class import Definitions + + +class Component(ABC): + name: ClassVar[Optional[str]] = None + + @classmethod + def registered_name(cls): + return cls.name or snakecase(cls.__name__) + + @classmethod + def generate_files(cls) -> None: + raise NotImplementedError() + + @abstractmethod + def build_defs(self) -> "Definitions": ... + + +def is_inside_deployment_project(path: str = ".") -> bool: + try: + _resolve_deployment_root_path(path) + return True + except DagsterError: + return False + + +def _resolve_deployment_root_path(path: str) -> str: + current_path = os.path.abspath(path) + while not _is_deployment_root(current_path): + current_path = os.path.dirname(current_path) + if current_path == "/": + raise DagsterError("Cannot find deployment root") + return current_path + + +def is_inside_code_location_project(path: str = ".") -> bool: + try: + _resolve_code_location_root_path(path) + return True + except DagsterError: + return False + + +def _resolve_code_location_root_path(path: str) -> str: + current_path = os.path.abspath(path) + while not _is_code_location_root(current_path): + current_path = os.path.dirname(current_path) + if current_path == "/": + raise DagsterError("Cannot find code location root") + return current_path + + +def _is_deployment_root(path: str) -> bool: + return os.path.exists(os.path.join(path, "code_locations")) + + +def _is_code_location_root(path: str) -> bool: + return os.path.basename(os.path.dirname(path)) == "code_locations" + + +# Deployment +_DEPLOYMENT_CODE_LOCATIONS_DIR: Final = "code_locations" + +# Code location +_CODE_LOCATION_CUSTOM_COMPONENTS_DIR: Final = "lib" +_CODE_LOCATION_COMPONENT_INSTANCES_DIR: Final = "components" + + +class DeploymentProjectContext: + @classmethod + def from_path(cls, path: str) -> Self: + return cls(root_path=_resolve_deployment_root_path(path)) + + def __init__(self, root_path: str): + self._root_path = root_path + + @property + def deployment_root(self) -> str: + return self._root_path + + @property + def code_location_root_path(self) -> str: + return os.path.join(self._root_path, _DEPLOYMENT_CODE_LOCATIONS_DIR) + + def has_code_location(self, name: str) -> bool: + return os.path.exists(os.path.join(self._root_path, "code_locations", name)) + + +class CodeLocationProjectContext: + @classmethod + def from_path(cls, path: str) -> Self: + root_path = _resolve_code_location_root_path(path) + name = os.path.basename(root_path) + component_registry = ComponentRegistry() + + # TODO: Rm when a more robust solution is implemented + # Make sure we can import from the cwd + if sys.path[0] != "": + sys.path.insert(0, "") + + components_lib_module = f"{name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}" + module = importlib.import_module(components_lib_module) + register_components_in_module(component_registry, module) + + return cls( + deployment_context=DeploymentProjectContext.from_path(path), + root_path=root_path, + name=os.path.basename(root_path), + component_registry=component_registry, + ) + + def __init__( + self, + deployment_context: DeploymentProjectContext, + root_path: str, + name: str, + component_registry: "ComponentRegistry", + ): + self._deployment_context = deployment_context + self._root_path = root_path + self._name = name + self._component_registry = component_registry + + @property + def deployment_context(self) -> DeploymentProjectContext: + return self._deployment_context + + @property + def component_types_root_path(self) -> str: + return os.path.join(self._root_path, self._name, _CODE_LOCATION_CUSTOM_COMPONENTS_DIR) + + @property + def component_types_root_module(self) -> str: + return f"{self._name}.{_CODE_LOCATION_CUSTOM_COMPONENTS_DIR}" + + def has_component_type(self, name: str) -> bool: + return self._component_registry.has(name) + + def get_component_type(self, name: str) -> Type[Component]: + if not self.has_component_type(name): + raise DagsterError(f"No component type named {name}") + return self._component_registry.get(name) + + @property + def component_instances_root_path(self) -> str: + return os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR) + + @property + def component_instances(self) -> Iterable[str]: + return os.listdir( + os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR) + ) + + def has_component_instance(self, name: str) -> bool: + return os.path.exists( + os.path.join(self._root_path, self._name, _CODE_LOCATION_COMPONENT_INSTANCES_DIR, name) + ) + + +class ComponentRegistry: + def __init__(self): + self._components: Dict[str, Type[Component]] = {} + + def register(self, name: str, component: Type[Component]) -> None: + self._components[name] = component + + def has(self, name: str) -> bool: + return name in self._components + + def get(self, name: str) -> Type[Component]: + return self._components[name] + + def keys(self) -> Iterable[str]: + return self._components.keys() + + def __repr__(self): + return f"" + + +def register_components_in_module(registry: ComponentRegistry, root_module: ModuleType) -> None: + from dagster._core.definitions.load_assets_from_modules import ( + find_modules_in_package, + find_subclasses_in_module, + ) + + for module in find_modules_in_package(root_module): + for component in find_subclasses_in_module(module, (Component,)): + if component is Component: + continue + name = f"{module.__name__}[{component.registered_name()}]" + registry.register(name, component) diff --git a/python_modules/dagster/dagster/_components/cli/__init__.py b/python_modules/dagster/dagster/_components/cli/__init__.py new file mode 100644 index 0000000000000..34fdd17211667 --- /dev/null +++ b/python_modules/dagster/dagster/_components/cli/__init__.py @@ -0,0 +1,28 @@ +import click + +from dagster._components.cli.generate import generate_cli +from dagster.version import __version__ + + +def create_dagster_components_cli(): + commands = { + "generate": generate_cli, + } + + @click.group( + commands=commands, + context_settings={"max_content_width": 120, "help_option_names": ["-h", "--help"]}, + ) + @click.version_option(__version__, "--version", "-v") + def group(): + """CLI tools for working with Dagster.""" + + return group + + +ENV_PREFIX = "DG_CLI" +cli = create_dagster_components_cli() + + +def main(): + cli(auto_envvar_prefix=ENV_PREFIX) diff --git a/python_modules/dagster/dagster/_components/cli/generate.py b/python_modules/dagster/dagster/_components/cli/generate.py new file mode 100644 index 0000000000000..49c7d9a651e09 --- /dev/null +++ b/python_modules/dagster/dagster/_components/cli/generate.py @@ -0,0 +1,102 @@ +import os +import sys + +import click + +from dagster._components import ( + CodeLocationProjectContext, + DeploymentProjectContext, + is_inside_code_location_project, + is_inside_deployment_project, +) +from dagster._generate.generate import ( + generate_code_location, + generate_component_instance, + generate_component_type, + generate_deployment, +) + + +@click.group(name="generate") +def generate_cli(): + """Commands for generating Dagster components and related entities.""" + + +@click.command(name="deployment") +@click.argument("path", type=str) +def generate_deployment_command(path: str) -> None: + """Generate a Dagster deployment instance.""" + dir_abspath = os.path.abspath(path) + if os.path.exists(dir_abspath): + click.echo( + click.style(f"A file or directory at {dir_abspath} already exists. ", fg="red") + + "\nPlease delete the contents of this path or choose another location." + ) + sys.exit(1) + generate_deployment(path) + + +@click.command(name="code-location") +@click.argument("name", type=str) +def generate_code_location_command(name: str) -> None: + """Generate a Dagster code location inside a component.""" + if not is_inside_deployment_project(): + click.echo( + click.style("This command must be run inside a Dagster deployment project.", fg="red") + ) + sys.exit(1) + + context = DeploymentProjectContext.from_path(os.getcwd()) + if context.has_code_location(name): + click.echo(click.style(f"A code location named {name} already exists.", fg="red")) + sys.exit(1) + + code_location_path = os.path.join(context.code_location_root_path, name) + generate_code_location(code_location_path) + + +@click.command(name="component-type") +@click.argument("name", type=str) +def generate_component_type_command(name: str) -> None: + """Generate a Dagster component instance.""" + if not is_inside_code_location_project(): + click.echo( + click.style( + "This command must be run inside a Dagster code location project.", fg="red" + ) + ) + sys.exit(1) + + context = CodeLocationProjectContext.from_path(os.getcwd()) + if context.has_component_type(name): + click.echo(click.style(f"A component type named `{name}` already exists.", fg="red")) + sys.exit(1) + + generate_component_type(context.component_types_root_path, name) + + +@click.command(name="component-instance") +@click.argument("component-type", type=str) +@click.argument("name", type=str) +def generate_component_instance_command(component_type: str, name: str) -> None: + """Generate a Dagster component instance.""" + if not is_inside_code_location_project(): + click.echo( + click.style( + "This command must be run inside a Dagster code location project.", fg="red" + ) + ) + sys.exit(1) + + context = CodeLocationProjectContext.from_path(os.getcwd()) + if not context.has_component_type(component_type): + click.echo( + click.style(f"No component type `{component_type}` could be resolved.", fg="red") + ) + sys.exit(1) + elif context.has_component_instance(name): + click.echo(click.style(f"A component instance named `{name}` already exists.", fg="red")) + sys.exit(1) + + component_type_cls = context.get_component_type(component_type) + generate_component_instance(context.component_instances_root_path, name, component_type_cls) diff --git a/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py b/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py index 4291f39488730..284d76795e505 100644 --- a/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py +++ b/python_modules/dagster/dagster/_core/definitions/load_assets_from_modules.py @@ -2,7 +2,7 @@ import pkgutil from importlib import import_module from types import ModuleType -from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, Union, cast +from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Set, Tuple, Type, Union, cast import dagster._check as check from dagster._core.definitions.asset_key import ( @@ -23,8 +23,11 @@ from dagster._core.errors import DagsterInvalidDefinitionError -def find_objects_in_module_of_types(module: ModuleType, types) -> Iterator: - """Yields objects of the given type(s).""" +def find_objects_in_module_of_types( + module: ModuleType, + types: Union[Type, Tuple[Type, ...]], +) -> Iterator: + """Yields instances or subclasses of the given type(s).""" for attr in dir(module): value = getattr(module, attr) if isinstance(value, types): @@ -33,6 +36,17 @@ def find_objects_in_module_of_types(module: ModuleType, types) -> Iterator: yield from value +def find_subclasses_in_module( + module: ModuleType, + types: Union[Type, Tuple[Type, ...]], +) -> Iterator: + """Yields instances or subclasses of the given type(s).""" + for attr in dir(module): + value = getattr(module, attr) + if isinstance(value, type) and issubclass(value, types): + yield value + + def assets_from_modules( modules: Iterable[ModuleType], extra_source_assets: Optional[Sequence[SourceAsset]] = None ) -> Tuple[Sequence[AssetsDefinition], Sequence[SourceAsset], Sequence[CacheableAssetsDefinition]]: diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index a3ce06873ea1b..9d84513afa691 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,10 +1,12 @@ import os import posixpath -from typing import List, Optional +from typing import Any, List, Optional, Type import click import jinja2 +from dagster._components import Component +from dagster._utils import camelcase, pushd from dagster.version import __version__ as dagster_version DEFAULT_EXCLUDES: List[str] = [ @@ -14,11 +16,57 @@ ".DS_Store", ".ruff_cache", "tox.ini", + ".gitkeep", # dummy file that allows empty directories to be checked into git ] PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" +def generate_deployment(path: str) -> None: + click.echo(f"Creating a Dagster deployment at {path}.") + + generate_project( + path=path, + name_placeholder="DEPLOYMENT_NAME_PLACEHOLDER", + templates_path=os.path.join( + os.path.dirname(__file__), "templates", "DEPLOYMENT_NAME_PLACEHOLDER" + ), + ) + + +def generate_code_location(path: str) -> None: + click.echo(f"Creating a Dagster code location at {path}.") + + generate_project( + path=path, + name_placeholder="CODE_LOCATION_NAME_PLACEHOLDER", + templates_path=os.path.join( + os.path.dirname(__file__), "templates", "CODE_LOCATION_NAME_PLACEHOLDER" + ), + ) + + +def generate_component_type(root_path: str, name: str) -> None: + click.echo(f"Creating a Dagster component type at {root_path}/{name}.py.") + + generate_project( + path=root_path, + name_placeholder="COMPONENT_TYPE_NAME_PLACEHOLDER", + templates_path=os.path.join(os.path.dirname(__file__), "templates", "COMPONENT_TYPE"), + project_name=name, + component_type_class_name=camelcase(name), + ) + + +def generate_component_instance(root_path: str, name: str, component_type: Type[Component]) -> None: + click.echo(f"Creating a Dagster component instance at {root_path}/{name}.py.") + + component_instance_root_path = os.path.join(root_path, name) + os.mkdir(component_instance_root_path) + with pushd(component_instance_root_path): + component_type.generate_files() + + def generate_repository(path: str): REPO_NAME_PLACEHOLDER = "REPO_NAME_PLACEHOLDER" @@ -40,6 +88,8 @@ def generate_project( excludes: Optional[List[str]] = None, name_placeholder: str = PROJECT_NAME_PLACEHOLDER, templates_path: str = PROJECT_NAME_PLACEHOLDER, + project_name: Optional[str] = None, + **other_template_vars: Any, ): """Renders templates for Dagster project.""" excludes = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes @@ -47,8 +97,9 @@ def generate_project( click.echo(f"Creating a Dagster project at {path}.") normalized_path = os.path.normpath(path) - project_name: str = os.path.basename(normalized_path).replace("-", "_") - os.mkdir(normalized_path) + project_name = project_name or os.path.basename(normalized_path).replace("-", "_") + if not os.path.exists(normalized_path): + os.mkdir(normalized_path) project_template_path: str = os.path.join( os.path.dirname(__file__), "templates", templates_path @@ -103,6 +154,7 @@ def generate_project( code_location_name=project_name, dagster_version=dagster_version, project_name=project_name, + **other_template_vars, ) ) f.write("\n") diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/components/.gitkeep b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/components/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/lib/__init__.py b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/lib/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja new file mode 100644 index 0000000000000..2545276208832 --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.jinja @@ -0,0 +1,24 @@ +[project] +name = "{{ project_name }}" +requires-python = ">=3.9,<3.13" +dependencies = [ + "dagster", + "dagster-cloud", +] + +[project.optional-dependencies] +dev = [ + "dagster-webserver", + "pytest", +] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "{{ project_name }}.definitions" +project_name = "{{ project_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ project_name }}_tests"] diff --git a/python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja b/python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja new file mode 100644 index 0000000000000..960cb28c5d708 --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja @@ -0,0 +1,10 @@ +from dagster import Component, Definitions + +class {{ component_type_class_name }}(Component): + + @classmethod + def generate_files(cls) -> None: + ... + + def build_defs(self) -> Definitions: + ... diff --git a/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/.github/workflows/dagster-cloud-deploy.yaml.jinja b/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/.github/workflows/dagster-cloud-deploy.yaml.jinja new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/code_locations/.gitkeep b/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/code_locations/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/dagster_cloud.yaml.jinja b/python_modules/dagster/dagster/_generate/templates/DEPLOYMENT_NAME_PLACEHOLDER/dagster_cloud.yaml.jinja new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_utils/__init__.py b/python_modules/dagster/dagster/_utils/__init__.py index 575796a7d9309..fce1525d93856 100644 --- a/python_modules/dagster/dagster/_utils/__init__.py +++ b/python_modules/dagster/dagster/_utils/__init__.py @@ -201,6 +201,14 @@ def camelcase(string: str) -> str: ) +def snakecase(string: str) -> str: + # Add an underscore before capital letters and lower the case + string = re.sub(r"(? Tuple[T, U]: check.mapping_param(ddict, "ddict") check.param_invariant(len(ddict) == 1, "ddict", "Expected dict with single item") diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py new file mode 100644 index 0000000000000..82e0504b52cdd --- /dev/null +++ b/python_modules/dagster/dagster_tests/cli_tests/test_generate_commands.py @@ -0,0 +1,161 @@ +import importlib +import os +import sys +from contextlib import contextmanager +from pathlib import Path + +from click.testing import CliRunner +from dagster._components import CodeLocationProjectContext +from dagster._components.cli.generate import ( + generate_code_location_command, + generate_component_instance_command, + generate_component_type_command, + generate_deployment_command, +) + + +def _ensure_cwd_on_sys_path(): + if sys.path[0] != "": + sys.path.insert(0, "") + + +def _assert_module_imports(module_name: str): + _ensure_cwd_on_sys_path() + assert importlib.import_module(module_name) + + +@contextmanager +def clean_module_cache(module_name: str): + prefix = f"{module_name}." + keys_to_del = { + key for key in sys.modules.keys() if key == module_name or key.startswith(prefix) + } + for key in keys_to_del: + del sys.modules[key] + yield + + +def test_generate_deployment_command_success(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(generate_deployment_command, ["foo"]) + assert result.exit_code == 0 + assert Path("foo").exists() + assert Path("foo/.github").exists() + assert Path("foo/.github/workflows").exists() + assert Path("foo/.github/workflows/dagster-cloud-deploy.yaml").exists() + assert Path("foo/dagster_cloud.yaml").exists() + assert Path("foo/code_locations").exists() + + +def test_generate_deployment_command_path_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + os.mkdir("foo") + result = runner.invoke(generate_deployment_command, ["foo"]) + assert result.exit_code != 0 + assert "already exists" in result.output + + +def test_generate_code_location_success(): + runner = CliRunner() + with runner.isolated_filesystem(): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code == 0 + assert Path("code_locations/bar").exists() + assert Path("code_locations/bar/bar").exists() + assert Path("code_locations/bar/bar/lib").exists() + assert Path("code_locations/bar/bar/components").exists() + assert Path("code_locations/bar/bar_tests").exists() + assert Path("code_locations/bar/pyproject.toml").exists() + + +def test_generate_code_location_outside_deployment_fails(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code != 0 + assert "must be run inside a Dagster deployment project" in result.output + + +def test_generate_code_location_already_exists_fails(): + runner = CliRunner() + with runner.isolated_filesystem(): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code == 0 + result = runner.invoke(generate_code_location_command, ["bar"]) + assert result.exit_code != 0 + assert "already exists" in result.output + + +def test_generate_component_type_success(): + runner = CliRunner() + with runner.isolated_filesystem(), clean_module_cache("bar"): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + # set up code location + runner.invoke(generate_code_location_command, ["bar"]) + os.chdir("code_locations/bar") + + result = runner.invoke(generate_component_type_command, ["baz"]) + assert result.exit_code == 0 + assert Path("bar/lib/baz.py").exists() + _assert_module_imports("bar.lib.baz") + context = CodeLocationProjectContext.from_path(os.getcwd()) + assert context.has_component_type("bar.lib.baz[baz]") + + +_SAMPLE_COMPONENT_TYPE = """ +from dagster import Component, Definitions, PipesSubprocessClient + +_SAMPLE_PIPES_SCRIPT = \""" +from dagster_pipes import open_dagster_pipes + +context = open_dagster_pipes() +context.report_asset_materialization({"alpha": "beta"}) +\""" + +class Baz(Component): + + @classmethod + def generate_files(cls): + with open("sample.py", "w") as f: + f.write(_SAMPLE_PIPES_SCRIPT) + + def build_defs(self) -> Definitions: + @asset + def foo(): + PipesSubprocessClient("foo.py").run(context, command=["python", "sample.py"]) + + return Definitions( + assets=[foo] + ) + +""" + + +def test_generate_component_instance_success(): + runner = CliRunner() + _ensure_cwd_on_sys_path() + with runner.isolated_filesystem(), clean_module_cache("bar"): + # set up deployment + runner.invoke(generate_deployment_command, ["foo"]) + os.chdir("foo") + # set up code location + runner.invoke(generate_code_location_command, ["bar"]) + os.chdir("code_locations/bar") + # set up component type + with open("bar/lib/baz.py", "w") as f: + f.write(_SAMPLE_COMPONENT_TYPE) + result = runner.invoke(generate_component_instance_command, ["bar.lib.baz[baz]", "qux"]) + assert result.exit_code == 0 + assert Path("bar/components/qux").exists() diff --git a/python_modules/dagster/dagster_tests/components_tests/__init__.py b/python_modules/dagster/dagster_tests/components_tests/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/setup.py b/python_modules/dagster/setup.py index 805126e175aca..2c79ba826eff2 100644 --- a/python_modules/dagster/setup.py +++ b/python_modules/dagster/setup.py @@ -167,6 +167,7 @@ def get_version() -> str: "console_scripts": [ "dagster = dagster.cli:main", "dagster-daemon = dagster.daemon.cli:main", + "dg = dagster._components.cli:main", ] }, )