diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..95223dc20eb57 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -117,7 +117,14 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -def scaffold_code_location_command(name: str): +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) +def scaffold_code_location_command(name: str, excludes: list): dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -126,7 +133,7 @@ def scaffold_code_location_command(name: str): ) sys.exit(1) - generate_code_location(dir_abspath) + generate_code_location(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..782480aecadf3 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import posixpath @@ -6,7 +8,13 @@ from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"] +IGNORE_PATTERN_LIST : list[str] = [ + "__pycache__", + ".pytest_cache", + "*.egg-info", + ".DS_Store", + "tox.ini", +] def generate_repository(path: str): @@ -26,7 +34,7 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): +def generate_code_location(path: str, excludes: list = []): CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster code location at {path}.") @@ -38,6 +46,7 @@ def generate_code_location(path: str): project_template_path=os.path.join( os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER ), + excludes=excludes, ) click.echo(f"Generated files for Dagster code location in {path}.") @@ -65,7 +74,11 @@ def generate_project(path: str): def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False + path: str, + name_placeholder: str, + project_template_path: str, + skip_mkdir: bool = False, + excludes: list[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -76,11 +89,13 @@ def _generate_files_from_template( loader = jinja2.FileSystemLoader(searchpath=project_template_path) env = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list + excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + if _should_skip_file(src_dir_path, excludes): continue src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) @@ -96,7 +111,7 @@ def _generate_files_from_template( # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue src_relative_file_path = os.path.relpath(src_file_path, project_template_path) @@ -124,13 +139,13 @@ def _generate_files_from_template( f.write("\n") -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: + for pattern in excludes: if pattern in path: return True diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 18a4302239867..837d3c638bf18 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,6 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] + [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 7a0fbf9055520..466f1e5ced3c6 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -76,6 +76,19 @@ def test_scaffold_code_location_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" +def test_scaffold_code_location_command_exclude_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + scaffold_code_location_command, + ["--name", "my_dagster_code", "--excludes", "setup*", "--excludes", "tests"], + ) + assert result.exit_code == 0 + assert not os.path.exists("my_dagster_code/setup.cfg") + assert not os.path.exists("my_dagster_code/setup.py") + assert not os.path.exists("my_dagster_code/tests/") + + def test_from_example_command_fails_when_example_not_available(): runner = CliRunner() with runner.isolated_filesystem():