feat(cli.project): enable optional minimalist configuration in templates

PEP 621 (Nov 2020) introduced pyproject.toml.
Setuptools is fully compatible with pyroject Completes dagster config in project template.
Minimalist config to make adopting dagster easy.

python_modules/dagster/dagster/_cli/project.py

@@ -117,7 +117,14 @@ def scaffold_repository_command(name: str):
     type=click.STRING,
     help="Name of the new Dagster code location",
 )
-def scaffold_code_location_command(name: str):
+@click.option(
+    '--skip-files',
+    multiple=True,
+    type=click.STRING,
+    default=[],
+    help="exclude file patterns from the project template",
+)
+def scaffold_code_location_command(name: str, skip_files: list):
     dir_abspath = os.path.abspath(name)
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
         click.echo(
@@ -126,7 +133,7 @@ def scaffold_code_location_command(name: str):
         )
         sys.exit(1)
 
-    generate_code_location(dir_abspath)
+    generate_code_location(dir_abspath, skip_files)
     click.echo(_styled_success_statement(name, dir_abspath))
 
 

python_modules/dagster/dagster/_generate/generate.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import os
 import posixpath
 
@@ -26,7 +27,7 @@ def generate_repository(path: str):
     click.echo(f"Generated files for Dagster repository in {path}.")
 
 
-def generate_code_location(path: str):
+def generate_code_location(path: str, skip_files: list = []):
     CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER"
 
     click.echo(f"Creating a Dagster code location at {path}.")
@@ -38,6 +39,7 @@ def generate_code_location(path: str):
         project_template_path=os.path.join(
             os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER
         ),
+        skip_files=skip_files,
     )
 
     click.echo(f"Generated files for Dagster code location in {path}.")
@@ -65,7 +67,11 @@ def generate_project(path: str):
 
 
 def _generate_files_from_template(
-    path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False
+    path: str,
+    name_placeholder: str,
+    project_template_path: str,
+    skip_mkdir: bool = False,
+    skip_files: list[str] = [],
 ):
     normalized_path = os.path.normpath(path)
     code_location_name = os.path.basename(normalized_path).replace("-", "_")
@@ -76,11 +82,13 @@ def _generate_files_from_template(
     loader = jinja2.FileSystemLoader(searchpath=project_template_path)
     env = jinja2.Environment(loader=loader)
 
+    # merge custom skip_files with the default list
+    ignore_patterns = _get_ignore_patterns(IGNORE_PATTERN_LIST, skip_files)
     for root, dirs, files in os.walk(project_template_path):
         # For each subdirectory in the source template, create a subdirectory in the destination.
         for dirname in dirs:
             src_dir_path = os.path.join(root, dirname)
-            if _should_skip_file(src_dir_path):
+            if _should_skip_file(src_dir_path, ignore_patterns):
                 continue
 
             src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path)
@@ -96,7 +104,7 @@ def _generate_files_from_template(
         # For each file in the source template, render a file in the destination.
         for filename in files:
             src_file_path = os.path.join(root, filename)
-            if _should_skip_file(src_file_path):
+            if _should_skip_file(src_file_path, ignore_patterns):
                 continue
 
             src_relative_file_path = os.path.relpath(src_file_path, project_template_path)
@@ -124,13 +132,20 @@ def _generate_files_from_template(
                 f.write("\n")
 
 
-def _should_skip_file(path):
+def _get_ignore_patterns(IGNORE_PATTERN_LIST, skip_files: list=[]) -> list[str]:
+    if skip_files:
+        return IGNORE_PATTERN_LIST.extend(skip_files)
+    else:
+        return IGNORE_PATTERN_LIST
+
+
+def _should_skip_file(path: str, ignore_patterns: list[str]):
     """Given a file path `path` in a source template, returns whether or not the file should be skipped
     when generating destination files.
 
     Technically, `path` could also be a directory path that should be skipped.
     """
-    for pattern in IGNORE_PATTERN_LIST:
+    for pattern in ignore_patterns:
         if pattern in path:
             return True
 

python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl

@@ -11,14 +11,17 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "dagster-webserver", 
+    "dagster-webserver",
     "pytest",
 ]
 
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.packages.find]
+exclude=["{{ code_location_name }}_tests"]
+
 [tool.dagster]
 module_name = "{{ code_location_name }}.definitions"
 code_location_name = "{{ code_location_name }}"