From 360df7d84b043616d5703c01e34204313c17ba46 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sat, 12 Oct 2024 13:00:37 +1100 Subject: [PATCH] feat(cli.project): enable optional minimalist configuration in templates PEP 621 (Nov 2020) introduced pyproject.toml. Setuptools is fully compatible with pyroject Completes dagster config in project template. Minimalist config to make adopting dagster easy. --- .../dagster/dagster/_cli/project.py | 11 +++++-- .../dagster/dagster/_generate/generate.py | 29 ++++++++++++++----- .../pyproject.toml.tmpl | 5 +++- .../cli_tests/test_project_commands.py | 13 +++++++++ 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..95223dc20eb57 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -117,7 +117,14 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -def scaffold_code_location_command(name: str): +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) +def scaffold_code_location_command(name: str, excludes: list): dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -126,7 +133,7 @@ def scaffold_code_location_command(name: str): ) sys.exit(1) - generate_code_location(dir_abspath) + generate_code_location(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..782480aecadf3 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import posixpath @@ -6,7 +8,13 @@ from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"] +IGNORE_PATTERN_LIST : list[str] = [ + "__pycache__", + ".pytest_cache", + "*.egg-info", + ".DS_Store", + "tox.ini", +] def generate_repository(path: str): @@ -26,7 +34,7 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): +def generate_code_location(path: str, excludes: list = []): CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster code location at {path}.") @@ -38,6 +46,7 @@ def generate_code_location(path: str): project_template_path=os.path.join( os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER ), + excludes=excludes, ) click.echo(f"Generated files for Dagster code location in {path}.") @@ -65,7 +74,11 @@ def generate_project(path: str): def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False + path: str, + name_placeholder: str, + project_template_path: str, + skip_mkdir: bool = False, + excludes: list[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -76,11 +89,13 @@ def _generate_files_from_template( loader = jinja2.FileSystemLoader(searchpath=project_template_path) env = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list + excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + if _should_skip_file(src_dir_path, excludes): continue src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) @@ -96,7 +111,7 @@ def _generate_files_from_template( # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue src_relative_file_path = os.path.relpath(src_file_path, project_template_path) @@ -124,13 +139,13 @@ def _generate_files_from_template( f.write("\n") -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: + for pattern in excludes: if pattern in path: return True diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 18a4302239867..837d3c638bf18 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,6 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] + [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 7a0fbf9055520..466f1e5ced3c6 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -76,6 +76,19 @@ def test_scaffold_code_location_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" +def test_scaffold_code_location_command_exclude_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + scaffold_code_location_command, + ["--name", "my_dagster_code", "--excludes", "setup*", "--excludes", "tests"], + ) + assert result.exit_code == 0 + assert not os.path.exists("my_dagster_code/setup.cfg") + assert not os.path.exists("my_dagster_code/setup.py") + assert not os.path.exists("my_dagster_code/tests/") + + def test_from_example_command_fails_when_example_not_available(): runner = CliRunner() with runner.isolated_filesystem():