From 2ec9386dc7915bf686ef441e314af8e34c167665 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sat, 12 Oct 2024 13:00:37 +1100 Subject: [PATCH] feat(cli.project): enable optional minimalist configuration in templates PEP 621 (Nov 2020) introduced pyproject.toml. Setuptools is fully compatible with pyroject Completes dagster config in project template. Minimalist config to make adopting dagster easy. --- .../dagster/dagster/_cli/project.py | 38 ++++++++----- .../dagster/dagster/_generate/__init__.py | 1 - .../dagster/dagster/_generate/generate.py | 56 +++++++++---------- .../pyproject.toml.tmpl | 5 +- .../cli_tests/test_project_commands.py | 25 +++++++++ 5 files changed, 77 insertions(+), 48 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..aa051604a2b35 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,13 +1,12 @@ import os import sys -from typing import NamedTuple, Optional, Sequence +from typing import NamedTuple, Optional, Sequence, Tuple, Union import click import requests from dagster._generate import ( download_example_from_github, - generate_code_location, generate_project, generate_repository, ) @@ -32,6 +31,7 @@ def project_cli(): ) scaffold_code_location_command_help_text = ( + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster code location, in the current directory. " "This CLI helps you to scaffold a new Dagster code location within a folder structure that " "includes multiple Dagster code locations." @@ -74,6 +74,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) +# start deprecated commands @project_cli.command( name="scaffold-repository", @@ -97,8 +98,7 @@ def scaffold_repository_command(name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`" - " instead.", + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", fg="yellow", ) ) @@ -118,16 +118,16 @@ def scaffold_repository_command(name: str): help="Name of the new Dagster code location", ) def scaffold_code_location_command(name: str): - dir_abspath = os.path.abspath(name) - if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): - click.echo( - click.style(f"The directory {dir_abspath} already exists. ", fg="red") - + "\nPlease delete the contents of this path or choose another location." - ) - sys.exit(1) + scaffold_command(name, excludes="README.md") - generate_code_location(dir_abspath) - click.echo(_styled_success_statement(name, dir_abspath)) + click.echo( + click.style( + "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.", + fg="yellow", + ) + ) + +# end deprecated commands def _check_and_error_on_package_conflicts(project_name: str) -> None: @@ -170,13 +170,21 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: type=click.STRING, help="Name of the new Dagster project", ) +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) @click.option( "--ignore-package-conflict", is_flag=True, default=False, help="Controls whether the project name can conflict with an existing PyPI package.", ) -def scaffold_command(name: str, ignore_package_conflict: bool): +def scaffold_command(name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool=False): + excludes = list(excludes) dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -188,7 +196,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool): if not ignore_package_conflict: _check_and_error_on_package_conflicts(name) - generate_project(dir_abspath) + generate_project(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/__init__.py b/python_modules/dagster/dagster/_generate/__init__.py index 680aa2bc669e5..f6d6b66c15d42 100644 --- a/python_modules/dagster/dagster/_generate/__init__.py +++ b/python_modules/dagster/dagster/_generate/__init__.py @@ -1,6 +1,5 @@ from dagster._generate.download import download_example_from_github as download_example_from_github from dagster._generate.generate import ( - generate_code_location as generate_code_location, generate_project as generate_project, generate_repository as generate_repository, ) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..03c16aa8c3dbc 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import posixpath @@ -6,7 +8,13 @@ from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"] +IGNORE_PATTERN_LIST : list[str] = [ + "__pycache__", + ".pytest_cache", + "*.egg-info", + ".DS_Store", + "tox.ini", +] def generate_repository(path: str): @@ -14,8 +22,8 @@ def generate_repository(path: str): click.echo(f"Creating a Dagster repository at {path}.") - # Step 1: Generate files for Dagster repository - _generate_files_from_template( + # Render templates for Dagster repository + _render_templates( path=path, name_placeholder=REPO_NAME_PLACEHOLDER, project_template_path=os.path.join( @@ -26,33 +34,13 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): - CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" - - click.echo(f"Creating a Dagster code location at {path}.") - - # Step 1: Generate files for Dagster code location including pyproject.toml, setup.py - _generate_files_from_template( - path=path, - name_placeholder=CODE_LOCATION_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER - ), - ) - - click.echo(f"Generated files for Dagster code location in {path}.") - - def generate_project(path: str): PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster project at {path}.") - # Step 1: Generate files for Dagster code location - generate_code_location(path) - - # Step 2: Generate project-level files, e.g. README - _generate_files_from_template( + # Step 1: Render templates for Dagster project + _render_templates( path=path, name_placeholder=PROJECT_NAME_PLACEHOLDER, project_template_path=os.path.join( @@ -64,8 +52,12 @@ def generate_project(path: str): click.echo(f"Generated files for Dagster project in {path}.") -def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False +def _render_templates( + path: str, + name_placeholder: str, + project_template_path: str, + skip_mkdir: bool = False, + excludes: list[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -76,11 +68,13 @@ def _generate_files_from_template( loader = jinja2.FileSystemLoader(searchpath=project_template_path) env = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list + excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + if _should_skip_file(src_dir_path, excludes): continue src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) @@ -96,7 +90,7 @@ def _generate_files_from_template( # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue src_relative_file_path = os.path.relpath(src_file_path, project_template_path) @@ -124,13 +118,13 @@ def _generate_files_from_template( f.write("\n") -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: + for pattern in excludes: if pattern in path: return True diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 18a4302239867..837d3c638bf18 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,6 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] + [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 7a0fbf9055520..b889bb81ec168 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -51,6 +51,21 @@ def test_project_scaffold_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_project.definitions" +def test_project_scaffold_command_excludes_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + scaffold_command, + ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], + ) + assert result.exit_code == 0 + assert os.path.exists("diet_dagster/pyproject.toml") + assert os.path.exists("diet_dagster/README.md") + assert not os.path.exists("diet_dagster/diet_dagster_tests/") + assert not os.path.exists("diet_dagster/setup.cfg") + assert not os.path.exists("diet_dagster/setup.py") + + def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): runner = CliRunner() with runner.isolated_filesystem(): @@ -76,6 +91,16 @@ def test_scaffold_code_location_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" +def test_scaffold_code_location_deprecation(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) + assert re.match( + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + result.output, + ) + + def test_from_example_command_fails_when_example_not_available(): runner = CliRunner() with runner.isolated_filesystem():