feat(cli.project): enable optional minimalist configuration in templates

python_modules/dagster/dagster/_cli/project.py

@@ -1,37 +1,34 @@
 import os
 import sys
-from typing import NamedTuple, Optional, Sequence
+from typing import List, NamedTuple, Optional, Sequence, Union
 
 import click
 import requests
 
-from dagster._generate import (
-    download_example_from_github,
-    generate_code_location,
-    generate_project,
-    generate_repository,
-)
+from dagster._generate import download_example_from_github, generate_project, generate_repository
 from dagster._generate.download import AVAILABLE_EXAMPLES
 from dagster.version import __version__ as dagster_version
 
 
 @click.group(name="project")
-def project_cli():
+def project_cli() -> None:
     """Commands for bootstrapping new Dagster projects and code locations."""
 
 
 # Keywords to flag in package names. When a project name contains one of these keywords, we check
 # if a conflicting PyPI package exists.
 FLAGGED_PACKAGE_KEYWORDS = ["dagster", "dbt"]
+VALID_EXCLUDES: list[str] = ["readme.md", "setup", "tests"]  # all lower case
 
 scaffold_repository_command_help_text = (
-    "(DEPRECATED; Use `dagster project scaffold-code-location` instead) "
+    "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) "
     "Create a folder structure with a single Dagster repository, in the current directory. "
     "This CLI helps you to scaffold a new Dagster repository within a folder structure that "
     "includes multiple Dagster repositories"
 )
 
 scaffold_code_location_command_help_text = (
+    "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) "
     "Create a folder structure with a single Dagster code location, in the current directory. "
     "This CLI helps you to scaffold a new Dagster code location within a folder structure that "
     "includes multiple Dagster code locations."
@@ -75,6 +72,37 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC
     return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False)
 
 
+def _check_and_error_on_package_conflicts(project_name: str) -> None:
+    package_check_result: PackageConflictCheckResult = check_if_pypi_package_conflict_exists(
+        project_name
+    )
+    if package_check_result.request_error_msg:
+        click.echo(
+            click.style(
+                "An error occurred while checking for package conflicts:"
+                f" {package_check_result.request_error_msg}. \n\nConflicting package names will"
+                " cause import errors in your project if the existing PyPI package is included"
+                " as a dependency in your scaffolded project. If desired, this check can be"
+                " skipped by adding the `--ignore-package-conflict` flag.",
+                fg="red",
+            )
+        )
+        sys.exit(1)
+
+    if package_check_result.conflict_exists:
+        click.echo(
+            click.style(
+                f"The project '{project_name}' conflicts with an existing PyPI package."
+                " Conflicting package names will cause import errors in your project if the"
+                " existing PyPI package is included as a dependency in your scaffolded"
+                " project. Please choose another name, or add the `--ignore-package-conflict`"
+                " flag to bypass this check.",
+                fg="yellow",
+            )
+        )
+        sys.exit(1)
+
+
 @project_cli.command(
     name="scaffold-repository",
     short_help=scaffold_repository_command_help_text,
@@ -86,7 +114,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC
     type=click.STRING,
     help="Name of the new Dagster repository",
 )
-def scaffold_repository_command(name: str):
+def scaffold_repository_command(name: str) -> None:
     dir_abspath = os.path.abspath(name)
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
         click.echo(
@@ -97,8 +125,7 @@ def scaffold_repository_command(name: str):
 
     click.echo(
         click.style(
-            "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`"
-            " instead.",
+            "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.",
             fg="yellow",
         )
     )
@@ -117,46 +144,15 @@ def scaffold_repository_command(name: str):
     type=click.STRING,
     help="Name of the new Dagster code location",
 )
-def scaffold_code_location_command(name: str):
-    dir_abspath = os.path.abspath(name)
-    if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
-        click.echo(
-            click.style(f"The directory {dir_abspath} already exists. ", fg="red")
-            + "\nPlease delete the contents of this path or choose another location."
-        )
-        sys.exit(1)
-
-    generate_code_location(dir_abspath)
-    click.echo(_styled_success_statement(name, dir_abspath))
-
-
-def _check_and_error_on_package_conflicts(project_name: str) -> None:
-    package_check_result = check_if_pypi_package_conflict_exists(project_name)
-    if package_check_result.request_error_msg:
-        click.echo(
-            click.style(
-                "An error occurred while checking for package conflicts:"
-                f" {package_check_result.request_error_msg}. \n\nConflicting package names will"
-                " cause import errors in your project if the existing PyPI package is included"
-                " as a dependency in your scaffolded project. If desired, this check can be"
-                " skipped by adding the `--ignore-package-conflict` flag.",
-                fg="red",
-            )
-        )
-        sys.exit(1)
-
-    if package_check_result.conflict_exists:
-        click.echo(
-            click.style(
-                f"The project '{project_name}' conflicts with an existing PyPI package."
-                " Conflicting package names will cause import errors in your project if the"
-                " existing PyPI package is included as a dependency in your scaffolded"
-                " project. Please choose another name, or add the `--ignore-package-conflict`"
-                " flag to bypass this check.",
-                fg="yellow",
-            )
+@click.pass_context
+def scaffold_code_location_command(context, name: str):
+    click.echo(
+        click.style(
+            "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.",
+            fg="yellow",
         )
-        sys.exit(1)
+    )
+    context.invoke(scaffold_command, name=name, excludes=["README.md"])
 
 
 @project_cli.command(
@@ -170,13 +166,24 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None:
     type=click.STRING,
     help="Name of the new Dagster project",
 )
+@click.option(
+    "--excludes",
+    multiple=True,
+    type=click.STRING,
+    default=[],
+    help=f"Exclude case-insensitive file patterns from the project template. Valid patterns: {VALID_EXCLUDES}",
+)
 @click.option(
     "--ignore-package-conflict",
     is_flag=True,
     default=False,
     help="Controls whether the project name can conflict with an existing PyPI package.",
 )
-def scaffold_command(name: str, ignore_package_conflict: bool):
+def scaffold_command(
+    name: str,
+    excludes: Optional[Union[List[str], tuple]] = None,
+    ignore_package_conflict: bool = False,
+) -> None:
     dir_abspath = os.path.abspath(name)
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
         click.echo(
@@ -185,10 +192,26 @@ def scaffold_command(name: str, ignore_package_conflict: bool):
         )
         sys.exit(1)
 
+    if isinstance(excludes, tuple):
+        excludes = list(excludes)
+    excludes = [] if not excludes else [item for item in excludes]
+
+    invalid_excludes = [item for item in excludes if item.lower() not in VALID_EXCLUDES]
+    if invalid_excludes:
+        click.echo(
+            click.style(
+                f"The following strings are invalid options for the excludes tag: {invalid_excludes}",
+                fg="red",
+            )
+            + f"Choose from {VALID_EXCLUDES}. Case-insensitive. "
+        )
+        sys.exit(1)
+    excludes = [item.lower() for item in excludes if item in VALID_EXCLUDES]
+
     if not ignore_package_conflict:
         _check_and_error_on_package_conflicts(name)
 
-    generate_project(dir_abspath)
+    generate_project(dir_abspath, excludes=excludes)
     click.echo(_styled_success_statement(name, dir_abspath))
 
 
@@ -219,7 +242,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool):
     default=dagster_version,
     show_default=True,
 )
-def from_example_command(name: Optional[str], example: str, version: str):
+def from_example_command(name: Optional[str], example: str, version: str) -> None:
     name = name or example
     dir_abspath = os.path.abspath(name) + "/"
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
@@ -229,7 +252,7 @@ def from_example_command(name: Optional[str], example: str, version: str):
         )
         sys.exit(1)
     else:
-        os.mkdir(dir_abspath)
+        os.makedirs(dir_abspath, exist_ok=True)
 
     download_example_from_github(dir_abspath, example, version)
 
@@ -241,7 +264,7 @@ def from_example_command(name: Optional[str], example: str, version: str):
     short_help=list_examples_command_help_text,
     help=list_examples_command_help_text,
 )
-def from_example_list_command():
+def from_example_list_command() -> None:
     click.echo("Examples available in `dagster project from-example`:")
 
     click.echo(_styled_list_examples_prints(AVAILABLE_EXAMPLES))
@@ -251,7 +274,7 @@ def _styled_list_examples_prints(examples: Sequence[str]) -> str:
     return "\n".join([f"* {name}" for name in examples])
 
 
-def _styled_success_statement(name: str, path: str):
+def _styled_success_statement(name: str, path: str) -> str:
     return (
         click.style("Success!", fg="green")
         + " Created "

python_modules/dagster/dagster/_generate/__init__.py

@@ -1,6 +1,5 @@
 from dagster._generate.download import download_example_from_github as download_example_from_github
 from dagster._generate.generate import (
-    generate_code_location as generate_code_location,
     generate_project as generate_project,
     generate_repository as generate_repository,
 )

python_modules/dagster/dagster/_generate/download.py

@@ -11,11 +11,13 @@
 # Examples aren't that can't be downloaded from the dagster project CLI
 EXAMPLES_TO_IGNORE = [
     "deploy_k8s_beta",
-    "docs_snippets",
     "docs_beta_snippets",
+    "docs_snippets",
     "experimental",
-    "temp_pins.txt",
     "use_case_repository",
+    "pyproject.toml",
+    "README.md",
+    "temp_pins.txt",
 ]
 # Hardcoded list of available examples. The list is tested against the examples folder in this mono
 # repo to make sure it's up-to-date.