diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..aa051604a2b35 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,13 +1,12 @@ import os import sys -from typing import NamedTuple, Optional, Sequence +from typing import NamedTuple, Optional, Sequence, Tuple, Union import click import requests from dagster._generate import ( download_example_from_github, - generate_code_location, generate_project, generate_repository, ) @@ -32,6 +31,7 @@ def project_cli(): ) scaffold_code_location_command_help_text = ( + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster code location, in the current directory. " "This CLI helps you to scaffold a new Dagster code location within a folder structure that " "includes multiple Dagster code locations." @@ -74,6 +74,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) +# start deprecated commands @project_cli.command( name="scaffold-repository", @@ -97,8 +98,7 @@ def scaffold_repository_command(name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`" - " instead.", + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", fg="yellow", ) ) @@ -118,16 +118,16 @@ def scaffold_repository_command(name: str): help="Name of the new Dagster code location", ) def scaffold_code_location_command(name: str): - dir_abspath = os.path.abspath(name) - if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): - click.echo( - click.style(f"The directory {dir_abspath} already exists. ", fg="red") - + "\nPlease delete the contents of this path or choose another location." - ) - sys.exit(1) + scaffold_command(name, excludes="README.md") - generate_code_location(dir_abspath) - click.echo(_styled_success_statement(name, dir_abspath)) + click.echo( + click.style( + "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.", + fg="yellow", + ) + ) + +# end deprecated commands def _check_and_error_on_package_conflicts(project_name: str) -> None: @@ -170,13 +170,21 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: type=click.STRING, help="Name of the new Dagster project", ) +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) @click.option( "--ignore-package-conflict", is_flag=True, default=False, help="Controls whether the project name can conflict with an existing PyPI package.", ) -def scaffold_command(name: str, ignore_package_conflict: bool): +def scaffold_command(name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool=False): + excludes = list(excludes) dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -188,7 +196,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool): if not ignore_package_conflict: _check_and_error_on_package_conflicts(name) - generate_project(dir_abspath) + generate_project(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/__init__.py b/python_modules/dagster/dagster/_generate/__init__.py index 680aa2bc669e5..f6d6b66c15d42 100644 --- a/python_modules/dagster/dagster/_generate/__init__.py +++ b/python_modules/dagster/dagster/_generate/__init__.py @@ -1,6 +1,5 @@ from dagster._generate.download import download_example_from_github as download_example_from_github from dagster._generate.generate import ( - generate_code_location as generate_code_location, generate_project as generate_project, generate_repository as generate_repository, ) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..03c16aa8c3dbc 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import posixpath @@ -6,7 +8,13 @@ from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"] +IGNORE_PATTERN_LIST : list[str] = [ + "__pycache__", + ".pytest_cache", + "*.egg-info", + ".DS_Store", + "tox.ini", +] def generate_repository(path: str): @@ -14,8 +22,8 @@ def generate_repository(path: str): click.echo(f"Creating a Dagster repository at {path}.") - # Step 1: Generate files for Dagster repository - _generate_files_from_template( + # Render templates for Dagster repository + _render_templates( path=path, name_placeholder=REPO_NAME_PLACEHOLDER, project_template_path=os.path.join( @@ -26,33 +34,13 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): - CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" - - click.echo(f"Creating a Dagster code location at {path}.") - - # Step 1: Generate files for Dagster code location including pyproject.toml, setup.py - _generate_files_from_template( - path=path, - name_placeholder=CODE_LOCATION_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER - ), - ) - - click.echo(f"Generated files for Dagster code location in {path}.") - - def generate_project(path: str): PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster project at {path}.") - # Step 1: Generate files for Dagster code location - generate_code_location(path) - - # Step 2: Generate project-level files, e.g. README - _generate_files_from_template( + # Step 1: Render templates for Dagster project + _render_templates( path=path, name_placeholder=PROJECT_NAME_PLACEHOLDER, project_template_path=os.path.join( @@ -64,8 +52,12 @@ def generate_project(path: str): click.echo(f"Generated files for Dagster project in {path}.") -def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False +def _render_templates( + path: str, + name_placeholder: str, + project_template_path: str, + skip_mkdir: bool = False, + excludes: list[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -76,11 +68,13 @@ def _generate_files_from_template( loader = jinja2.FileSystemLoader(searchpath=project_template_path) env = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list + excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + if _should_skip_file(src_dir_path, excludes): continue src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) @@ -96,7 +90,7 @@ def _generate_files_from_template( # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue src_relative_file_path = os.path.relpath(src_file_path, project_template_path) @@ -124,13 +118,13 @@ def _generate_files_from_template( f.write("\n") -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: + for pattern in excludes: if pattern in path: return True diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 18a4302239867..837d3c638bf18 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,6 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] + [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 7a0fbf9055520..b889bb81ec168 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -51,6 +51,21 @@ def test_project_scaffold_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_project.definitions" +def test_project_scaffold_command_excludes_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + scaffold_command, + ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], + ) + assert result.exit_code == 0 + assert os.path.exists("diet_dagster/pyproject.toml") + assert os.path.exists("diet_dagster/README.md") + assert not os.path.exists("diet_dagster/diet_dagster_tests/") + assert not os.path.exists("diet_dagster/setup.cfg") + assert not os.path.exists("diet_dagster/setup.py") + + def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): runner = CliRunner() with runner.isolated_filesystem(): @@ -76,6 +91,16 @@ def test_scaffold_code_location_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" +def test_scaffold_code_location_deprecation(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) + assert re.match( + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + result.output, + ) + + def test_from_example_command_fails_when_example_not_available(): runner = CliRunner() with runner.isolated_filesystem():