diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..a026849fea4b0 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -117,7 +117,14 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -def scaffold_code_location_command(name: str): +@click.option( + '--excludes', + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) +def scaffold_code_location_command(name: str, excludes: list): dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -126,7 +133,7 @@ def scaffold_code_location_command(name: str): ) sys.exit(1) - generate_code_location(dir_abspath) + generate_code_location(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..7087d97ff80d7 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,3 +1,4 @@ +from __future__ import annotations import os import posixpath @@ -26,7 +27,7 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): +def generate_code_location(path: str, excludes: list = []): CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster code location at {path}.") @@ -38,6 +39,7 @@ def generate_code_location(path: str): project_template_path=os.path.join( os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER ), + excludes=excludes, ) click.echo(f"Generated files for Dagster code location in {path}.") @@ -65,7 +67,11 @@ def generate_project(path: str): def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False + path: str, + name_placeholder: str, + project_template_path: str, + skip_mkdir: bool = False, + excludes: list[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -76,11 +82,13 @@ def _generate_files_from_template( loader = jinja2.FileSystemLoader(searchpath=project_template_path) env = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list + excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + if _should_skip_file(src_dir_path, excludes): continue src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) @@ -96,7 +104,7 @@ def _generate_files_from_template( # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue src_relative_file_path = os.path.relpath(src_file_path, project_template_path) @@ -124,13 +132,13 @@ def _generate_files_from_template( f.write("\n") -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: list[str]): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: + for pattern in excludes: if pattern in path: return True diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 18a4302239867..837d3c638bf18 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,6 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] + [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}"