feat(cli.project): enable optional minimalist configuration in templa…

…tes (#25245)

PEP 621 (Nov 2020) introduced pyproject.toml.
Setuptools is fully compatible with pyroject. 
All the config in setup.cfg and setup.py will also be in pyproject.toml
project template if this is merged.
Minimalist config to make adopting dagster easy.

## Summary & Motivation
See #25244 

## How I Tested These Changes

![image](https://github.com/user-attachments/assets/75473daa-a282-4564-93fd-db1d95cf2691)
Last time I tried the run the test suite it failed because my macbook
has an intel chipset.

## Changelog
(CLI) 
`dagster project scaffold`: Add option to create dagster projects from
templates with excluded files/filepaths
`dagster project scaffold-code-location`: Refactored to leverage
scaffold command with excludes flag. Deprecated.

---------

Co-authored-by: Colton Padden <[email protected]>

python_modules/dagster/dagster/_cli/project.py

@@ -1,37 +1,34 @@
 import os
 import sys
-from typing import NamedTuple, Optional, Sequence
+from typing import List, NamedTuple, Optional, Sequence, Union
 
 import click
 import requests
 
-from dagster._generate import (
-    download_example_from_github,
-    generate_code_location,
-    generate_project,
-    generate_repository,
-)
+from dagster._generate import download_example_from_github, generate_project, generate_repository
 from dagster._generate.download import AVAILABLE_EXAMPLES
 from dagster.version import __version__ as dagster_version
 
 
 @click.group(name="project")
-def project_cli():
+def project_cli() -> None:
     """Commands for bootstrapping new Dagster projects and code locations."""
 
 
 # Keywords to flag in package names. When a project name contains one of these keywords, we check
 # if a conflicting PyPI package exists.
 FLAGGED_PACKAGE_KEYWORDS = ["dagster", "dbt"]
+VALID_EXCLUDES: list[str] = ["readme.md", "setup", "tests"]  # all lower case
 
 scaffold_repository_command_help_text = (
-    "(DEPRECATED; Use `dagster project scaffold-code-location` instead) "
+    "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) "
     "Create a folder structure with a single Dagster repository, in the current directory. "
     "This CLI helps you to scaffold a new Dagster repository within a folder structure that "
     "includes multiple Dagster repositories"
 )
 
 scaffold_code_location_command_help_text = (
+    "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) "
     "Create a folder structure with a single Dagster code location, in the current directory. "
     "This CLI helps you to scaffold a new Dagster code location within a folder structure that "
     "includes multiple Dagster code locations."
@@ -75,6 +72,37 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC
     return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False)
 
 
+def _check_and_error_on_package_conflicts(project_name: str) -> None:
+    package_check_result: PackageConflictCheckResult = check_if_pypi_package_conflict_exists(
+        project_name
+    )
+    if package_check_result.request_error_msg:
+        click.echo(
+            click.style(
+                "An error occurred while checking for package conflicts:"
+                f" {package_check_result.request_error_msg}. \n\nConflicting package names will"
+                " cause import errors in your project if the existing PyPI package is included"
+                " as a dependency in your scaffolded project. If desired, this check can be"
+                " skipped by adding the `--ignore-package-conflict` flag.",
+                fg="red",
+            )
+        )
+        sys.exit(1)
+
+    if package_check_result.conflict_exists:
+        click.echo(
+            click.style(
+                f"The project '{project_name}' conflicts with an existing PyPI package."
+                " Conflicting package names will cause import errors in your project if the"
+                " existing PyPI package is included as a dependency in your scaffolded"
+                " project. Please choose another name, or add the `--ignore-package-conflict`"
+                " flag to bypass this check.",
+                fg="yellow",
+            )
+        )
+        sys.exit(1)
+
+
 @project_cli.command(
     name="scaffold-repository",
     short_help=scaffold_repository_command_help_text,
@@ -86,7 +114,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC
     type=click.STRING,
     help="Name of the new Dagster repository",
 )
-def scaffold_repository_command(name: str):
+def scaffold_repository_command(name: str) -> None:
     dir_abspath = os.path.abspath(name)
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
         click.echo(
@@ -97,8 +125,7 @@ def scaffold_repository_command(name: str):
 
     click.echo(
         click.style(
-            "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`"
-            " instead.",
+            "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.",
             fg="yellow",
         )
     )
@@ -117,46 +144,15 @@ def scaffold_repository_command(name: str):
     type=click.STRING,
     help="Name of the new Dagster code location",
 )
-def scaffold_code_location_command(name: str):
-    dir_abspath = os.path.abspath(name)
-    if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
-        click.echo(
-            click.style(f"The directory {dir_abspath} already exists. ", fg="red")
-            + "\nPlease delete the contents of this path or choose another location."
-        )
-        sys.exit(1)
-
-    generate_code_location(dir_abspath)
-    click.echo(_styled_success_statement(name, dir_abspath))
-
-
-def _check_and_error_on_package_conflicts(project_name: str) -> None:
-    package_check_result = check_if_pypi_package_conflict_exists(project_name)
-    if package_check_result.request_error_msg:
-        click.echo(
-            click.style(
-                "An error occurred while checking for package conflicts:"
-                f" {package_check_result.request_error_msg}. \n\nConflicting package names will"
-                " cause import errors in your project if the existing PyPI package is included"
-                " as a dependency in your scaffolded project. If desired, this check can be"
-                " skipped by adding the `--ignore-package-conflict` flag.",
-                fg="red",
-            )
-        )
-        sys.exit(1)
-
-    if package_check_result.conflict_exists:
-        click.echo(
-            click.style(
-                f"The project '{project_name}' conflicts with an existing PyPI package."
-                " Conflicting package names will cause import errors in your project if the"
-                " existing PyPI package is included as a dependency in your scaffolded"
-                " project. Please choose another name, or add the `--ignore-package-conflict`"
-                " flag to bypass this check.",
-                fg="yellow",
-            )
+@click.pass_context
+def scaffold_code_location_command(context, name: str):
+    click.echo(
+        click.style(
+            "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.",
+            fg="yellow",
         )
-        sys.exit(1)
+    )
+    context.invoke(scaffold_command, name=name, excludes=["README.md"])
 
 
 @project_cli.command(
@@ -170,13 +166,24 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None:
     type=click.STRING,
     help="Name of the new Dagster project",
 )
+@click.option(
+    "--excludes",
+    multiple=True,
+    type=click.STRING,
+    default=[],
+    help=f"Exclude case-insensitive file patterns from the project template. Valid patterns: {VALID_EXCLUDES}",
+)
 @click.option(
     "--ignore-package-conflict",
     is_flag=True,
     default=False,
     help="Controls whether the project name can conflict with an existing PyPI package.",
 )
-def scaffold_command(name: str, ignore_package_conflict: bool):
+def scaffold_command(
+    name: str,
+    excludes: Optional[Union[List[str], tuple]] = None,
+    ignore_package_conflict: bool = False,
+) -> None:
     dir_abspath = os.path.abspath(name)
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
         click.echo(
@@ -185,10 +192,26 @@ def scaffold_command(name: str, ignore_package_conflict: bool):
         )
         sys.exit(1)
 
+    if isinstance(excludes, tuple):
+        excludes = list(excludes)
+    excludes = [] if not excludes else [item for item in excludes]
+
+    invalid_excludes = [item for item in excludes if item.lower() not in VALID_EXCLUDES]
+    if invalid_excludes:
+        click.echo(
+            click.style(
+                f"The following strings are invalid options for the excludes tag: {invalid_excludes}",
+                fg="red",
+            )
+            + f"Choose from {VALID_EXCLUDES}. Case-insensitive. "
+        )
+        sys.exit(1)
+    excludes = [item.lower() for item in excludes if item in VALID_EXCLUDES]
+
     if not ignore_package_conflict:
         _check_and_error_on_package_conflicts(name)
 
-    generate_project(dir_abspath)
+    generate_project(dir_abspath, excludes=excludes)
     click.echo(_styled_success_statement(name, dir_abspath))
 
 
@@ -219,7 +242,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool):
     default=dagster_version,
     show_default=True,
 )
-def from_example_command(name: Optional[str], example: str, version: str):
+def from_example_command(name: Optional[str], example: str, version: str) -> None:
     name = name or example
     dir_abspath = os.path.abspath(name) + "/"
     if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath):
@@ -229,7 +252,7 @@ def from_example_command(name: Optional[str], example: str, version: str):
         )
         sys.exit(1)
     else:
-        os.mkdir(dir_abspath)
+        os.makedirs(dir_abspath, exist_ok=True)
 
     download_example_from_github(dir_abspath, example, version)
 
@@ -241,7 +264,7 @@ def from_example_command(name: Optional[str], example: str, version: str):
     short_help=list_examples_command_help_text,
     help=list_examples_command_help_text,
 )
-def from_example_list_command():
+def from_example_list_command() -> None:
     click.echo("Examples available in `dagster project from-example`:")
 
     click.echo(_styled_list_examples_prints(AVAILABLE_EXAMPLES))
@@ -251,7 +274,7 @@ def _styled_list_examples_prints(examples: Sequence[str]) -> str:
     return "\n".join([f"* {name}" for name in examples])
 
 
-def _styled_success_statement(name: str, path: str):
+def _styled_success_statement(name: str, path: str) -> str:
     return (
         click.style("Success!", fg="green")
         + " Created "

python_modules/dagster/dagster/_generate/__init__.py

@@ -1,6 +1,5 @@
 from dagster._generate.download import download_example_from_github as download_example_from_github
 from dagster._generate.generate import (
-    generate_code_location as generate_code_location,
     generate_project as generate_project,
     generate_repository as generate_repository,
 )

python_modules/dagster/dagster/_generate/download.py

@@ -11,11 +11,13 @@
 # Examples aren't that can't be downloaded from the dagster project CLI
 EXAMPLES_TO_IGNORE = [
     "deploy_k8s_beta",
-    "docs_snippets",
     "docs_beta_snippets",
+    "docs_snippets",
     "experimental",
-    "temp_pins.txt",
     "use_case_repository",
+    "pyproject.toml",
+    "README.md",
+    "temp_pins.txt",
 ]
 # Hardcoded list of available examples. The list is tested against the examples folder in this mono
 # repo to make sure it's up-to-date.