From e916af9dfd0d66bb78887ca1237f5b2bad29b993 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sat, 19 Oct 2024 23:28:22 +1100 Subject: [PATCH] feat(cli): deprecate project scaffold-code-location Eliminated code duplication between `dagster project scaffold` and `dagster project scaffold-code-location`. Turns out the only difference is that scaffold-code-location excludes the readme. This confused me for hours. --- .../dagster/dagster/_cli/project.py | 34 ++++++++----------- .../dagster/dagster/_generate/download.py | 1 + .../dagster/dagster/_generate/generate.py | 10 +++--- .../PROJECT_NAME_PLACEHOLDER}/__init__.py | 0 .../PROJECT_NAME_PLACEHOLDER}/assets.py | 0 .../PROJECT_NAME_PLACEHOLDER}/definitions.py | 0 .../__init__.py | 0 .../test_assets.py.tmpl | 0 .../README.md | 0 .../pyproject.toml.tmpl | 0 .../setup.cfg.tmpl | 0 .../setup.py.tmpl | 0 .../cli_tests/test_project_commands.py | 10 +++--- 13 files changed, 26 insertions(+), 29 deletions(-) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/__init__.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/assets.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/definitions.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests => PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests}/__init__.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests => PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests}/test_assets.py.tmpl (100%) rename python_modules/dagster/dagster/_generate/templates/{PROJECT_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER}/README.md (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER}/pyproject.toml.tmpl (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER}/setup.cfg.tmpl (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_PLACEHOLDER}/setup.py.tmpl (100%) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index 95223dc20eb57..6219c1039a040 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -32,6 +32,7 @@ def project_cli(): ) scaffold_code_location_command_help_text = ( + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster code location, in the current directory. " "This CLI helps you to scaffold a new Dagster code location within a folder structure that " "includes multiple Dagster code locations." @@ -74,6 +75,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) +# start deprecated commands @project_cli.command( name="scaffold-repository", @@ -117,24 +119,10 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -@click.option( - "--excludes", - multiple=True, - type=click.STRING, - default=[], - help="Exclude file patterns from the project template", -) -def scaffold_code_location_command(name: str, excludes: list): - dir_abspath = os.path.abspath(name) - if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): - click.echo( - click.style(f"The directory {dir_abspath} already exists. ", fg="red") - + "\nPlease delete the contents of this path or choose another location." - ) - sys.exit(1) +def scaffold_code_location_command(name: str): + scaffold_command(name, excludes="README.md") - generate_code_location(dir_abspath, excludes) - click.echo(_styled_success_statement(name, dir_abspath)) +# end deprecated commands def _check_and_error_on_package_conflicts(project_name: str) -> None: @@ -177,13 +165,21 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: type=click.STRING, help="Name of the new Dagster project", ) +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) @click.option( "--ignore-package-conflict", is_flag=True, default=False, help="Controls whether the project name can conflict with an existing PyPI package.", ) -def scaffold_command(name: str, ignore_package_conflict: bool): +def scaffold_command(name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool=False): + excludes = list(excludes) dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -195,7 +191,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool): if not ignore_package_conflict: _check_and_error_on_package_conflicts(name) - generate_project(dir_abspath) + generate_project(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index d51fb3f81a660..9a14877494be1 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -30,6 +30,7 @@ "deploy_ecs", "deploy_k8s", "development_to_production", + "etl_tutorial", "feature_graph_backed_assets", "getting_started_etl_tutorial", "project_analytics", diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 782480aecadf3..cebb6dc3789d5 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -8,7 +8,7 @@ from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST : list[str] = [ +IGNORE_PATTERN_LIST: list[str] = [ "__pycache__", ".pytest_cache", "*.egg-info", @@ -18,6 +18,7 @@ def generate_repository(path: str): + """Part of the deprecated scaffold-repository command""" REPO_NAME_PLACEHOLDER = "REPO_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster repository at {path}.") @@ -80,6 +81,7 @@ def _generate_files_from_template( skip_mkdir: bool = False, excludes: list[str] = [], ): + """Renders Jinja2 templates to the dagster project.""" normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -145,8 +147,4 @@ def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): Technically, `path` could also be a directory path that should be skipped. """ - for pattern in excludes: - if pattern in path: - return True - - return False + return any(pattern in path for pattern in excludes) diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.tmpl similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.tmpl diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/README.md similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/README.md diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/pyproject.toml.tmpl similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/pyproject.toml.tmpl diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/setup.cfg.tmpl similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/setup.cfg.tmpl diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/setup.py.tmpl similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_PLACEHOLDER/setup.py.tmpl diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 466f1e5ced3c6..788dd4d978491 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -81,12 +81,14 @@ def test_scaffold_code_location_command_exclude_succeeds(): with runner.isolated_filesystem(): result = runner.invoke( scaffold_code_location_command, - ["--name", "my_dagster_code", "--excludes", "setup*", "--excludes", "tests"], + ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], ) assert result.exit_code == 0 - assert not os.path.exists("my_dagster_code/setup.cfg") - assert not os.path.exists("my_dagster_code/setup.py") - assert not os.path.exists("my_dagster_code/tests/") + assert os.path.exists("diet_dagster/pyproject.toml") + assert os.path.exists("diet_dagster/README.md") + assert not os.path.exists("diet_dagster/diet_dagster_tests/") + assert not os.path.exists("diet_dagster/setup.cfg") + assert not os.path.exists("diet_dagster/setup.py") def test_from_example_command_fails_when_example_not_available():