feat(cli.project): enable optional minimalist configuration in templates

PEP 621 (Nov 2020) introduced pyproject.toml.
Setuptools is fully compatible with pyroject Completes dagster config in project template.
Minimalist config to make adopting dagster easy.

python_modules/dagster/dagster/_cli/project.py

@@ -1,13 +1,12 @@
 import os
 import sys
-from typing import NamedTuple, Optional, Sequence
+from typing import NamedTuple, Optional, Sequence, Tuple, Union
 
 import click
 import requests
 
 from dagster._generate import (
     download_example_from_github,
-    generate_code_location,
     generate_project,
     generate_repository,
 )
@@ -32,6 +31,7 @@ def project_cli():
 )
 
 scaffold_code_location_command_help_text = (
+    "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) "
     "Create a folder structure with a single Dagster code location, in the current directory. "
     "This CLI helps you to scaffold a new Dagster code location within a folder structure that "
     "includes multiple Dagster code locations."
@@ -74,6 +74,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC
 
     return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False)
 
+# start deprecated commands
 
 @project_cli.command(
     name="scaffold-repository",
@@ -97,8 +98,7 @@ def scaffold_repository_command(name: str):
 
     click.echo(
         click.style(
-            "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`"
-            " instead.",
+            "WARNING: This command is deprecated. Use `dagster project scaffold` instead.",
             fg="yellow",
         )
     )
@@ -118,16 +118,16 @@ def scaffold_repository_command(name: str):
     help="Name of the new Dagster code location",
 )
 def scaffold_code_location_command(name: str):
-    dir_abspath = os.path.abspath(name)
-    if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
-        click.echo(
-            click.style(f"The directory {dir_abspath} already exists. ", fg="red")
-            + "\nPlease delete the contents of this path or choose another location."
-        )
-        sys.exit(1)
+    scaffold_command(name, excludes="README.md")
 
-    generate_code_location(dir_abspath)
-    click.echo(_styled_success_statement(name, dir_abspath))
+    click.echo(
+        click.style(
+            "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.",
+            fg="yellow",
+        )
+    )
+
+# end deprecated commands
 
 
 def _check_and_error_on_package_conflicts(project_name: str) -> None:
@@ -170,13 +170,21 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None:
     type=click.STRING,
     help="Name of the new Dagster project",
 )
+@click.option(
+    "--excludes",
+    multiple=True,
+    type=click.STRING,
+    default=[],
+    help="Exclude file patterns from the project template",
+)
 @click.option(
     "--ignore-package-conflict",
     is_flag=True,
     default=False,
     help="Controls whether the project name can conflict with an existing PyPI package.",
 )
-def scaffold_command(name: str, ignore_package_conflict: bool):
+def scaffold_command(name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool=False):
+    excludes = list(excludes)
     dir_abspath = os.path.abspath(name)
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
         click.echo(
@@ -188,7 +196,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool):
     if not ignore_package_conflict:
         _check_and_error_on_package_conflicts(name)
 
-    generate_project(dir_abspath)
+    generate_project(dir_abspath, excludes)
     click.echo(_styled_success_statement(name, dir_abspath))
 
 

python_modules/dagster/dagster/_generate/__init__.py

@@ -1,6 +1,5 @@
 from dagster._generate.download import download_example_from_github as download_example_from_github
 from dagster._generate.generate import (
-    generate_code_location as generate_code_location,
     generate_project as generate_project,
     generate_repository as generate_repository,
 )

python_modules/dagster/dagster/_generate/generate.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import posixpath
 
@@ -6,16 +8,22 @@
 
 from dagster.version import __version__ as dagster_version
 
-IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"]
+IGNORE_PATTERN_LIST : list[str] = [
+    "__pycache__",
+    ".pytest_cache",
+    "*.egg-info",
+    ".DS_Store",
+    "tox.ini",
+]
 
 
 def generate_repository(path: str):
     REPO_NAME_PLACEHOLDER = "REPO_NAME_PLACEHOLDER"
 
     click.echo(f"Creating a Dagster repository at {path}.")
 
-    # Step 1: Generate files for Dagster repository
-    _generate_files_from_template(
+    # Render templates for Dagster repository
+    _render_templates(
         path=path,
         name_placeholder=REPO_NAME_PLACEHOLDER,
         project_template_path=os.path.join(
@@ -26,33 +34,13 @@ def generate_repository(path: str):
     click.echo(f"Generated files for Dagster repository in {path}.")
 
 
-def generate_code_location(path: str):
-    CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER"
-
-    click.echo(f"Creating a Dagster code location at {path}.")
-
-    # Step 1: Generate files for Dagster code location including pyproject.toml, setup.py
-    _generate_files_from_template(
-        path=path,
-        name_placeholder=CODE_LOCATION_NAME_PLACEHOLDER,
-        project_template_path=os.path.join(
-            os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER
-        ),
-    )
-
-    click.echo(f"Generated files for Dagster code location in {path}.")
-
-
 def generate_project(path: str):
     PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER"
 
     click.echo(f"Creating a Dagster project at {path}.")
 
-    # Step 1: Generate files for Dagster code location
-    generate_code_location(path)
-
-    # Step 2: Generate project-level files, e.g. README
-    _generate_files_from_template(
+    # Step 1: Render templates for Dagster project
+    _render_templates(
         path=path,
         name_placeholder=PROJECT_NAME_PLACEHOLDER,
         project_template_path=os.path.join(
@@ -64,8 +52,12 @@ def generate_project(path: str):
     click.echo(f"Generated files for Dagster project in {path}.")
 
 
-def _generate_files_from_template(
-    path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False
+def _render_templates(
+    path: str,
+    name_placeholder: str,
+    project_template_path: str,
+    skip_mkdir: bool = False,
+    excludes: list[str] = [],
 ):
     normalized_path = os.path.normpath(path)
     code_location_name = os.path.basename(normalized_path).replace("-", "_")
@@ -76,11 +68,13 @@ def _generate_files_from_template(
     loader = jinja2.FileSystemLoader(searchpath=project_template_path)
     env = jinja2.Environment(loader=loader)
 
+    # merge custom skip_files with the default list
+    excludes = IGNORE_PATTERN_LIST + excludes
     for root, dirs, files in os.walk(project_template_path):
         # For each subdirectory in the source template, create a subdirectory in the destination.
         for dirname in dirs:
             src_dir_path = os.path.join(root, dirname)
-            if _should_skip_file(src_dir_path):
+            if _should_skip_file(src_dir_path, excludes):
                 continue
 
             src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path)
@@ -96,7 +90,7 @@ def _generate_files_from_template(
         # For each file in the source template, render a file in the destination.
         for filename in files:
             src_file_path = os.path.join(root, filename)
-            if _should_skip_file(src_file_path):
+            if _should_skip_file(src_file_path, excludes):
                 continue
 
             src_relative_file_path = os.path.relpath(src_file_path, project_template_path)
@@ -124,13 +118,13 @@ def _generate_files_from_template(
                 f.write("\n")
 
 
-def _should_skip_file(path):
+def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST):
     """Given a file path `path` in a source template, returns whether or not the file should be skipped
     when generating destination files.
 
     Technically, `path` could also be a directory path that should be skipped.
     """
-    for pattern in IGNORE_PATTERN_LIST:
+    for pattern in excludes:
         if pattern in path:
             return True
 

python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl

@@ -11,14 +11,17 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "dagster-webserver", 
+    "dagster-webserver",
     "pytest",
 ]
 
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.packages.find]
+exclude=["{{ code_location_name }}_tests"]
+
 [tool.dagster]
 module_name = "{{ code_location_name }}.definitions"
 code_location_name = "{{ code_location_name }}"

python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py

@@ -51,6 +51,21 @@ def test_project_scaffold_command_succeeds():
         assert origins[0].loadable_target_origin.module_name == "my_dagster_project.definitions"
 
 
+def test_project_scaffold_command_excludes_succeeds():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            scaffold_command,
+            ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"],
+        )
+        assert result.exit_code == 0
+        assert os.path.exists("diet_dagster/pyproject.toml")
+        assert os.path.exists("diet_dagster/README.md")
+        assert not os.path.exists("diet_dagster/diet_dagster_tests/")
+        assert not os.path.exists("diet_dagster/setup.cfg")
+        assert not os.path.exists("diet_dagster/setup.py")
+
+
 def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists():
     runner = CliRunner()
     with runner.isolated_filesystem():
@@ -76,6 +91,16 @@ def test_scaffold_code_location_command_succeeds():
         assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions"
 
 
+def test_scaffold_code_location_deprecation():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"])
+        assert re.match(
+            "WARNING: This command is deprecated. Use `dagster project scaffold` instead.",
+            result.output,
+        )
+
+
 def test_from_example_command_fails_when_example_not_available():
     runner = CliRunner()
     with runner.isolated_filesystem():