From 270b21c9fcfceda09559977e765957abb902e859 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sat, 12 Oct 2024 13:00:37 +1100 Subject: [PATCH 1/8] feat(cli.project): enable optional minimalist configuration in templates PEP 621 (Nov 2020) introduced pyproject.toml. Setuptools is fully compatible with pyroject Completes dagster config in project template. Minimalist config to make adopting dagster easy. --- .../dagster/dagster/_cli/project.py | 38 ++++++++----- .../dagster/dagster/_generate/__init__.py | 1 - .../dagster/dagster/_generate/generate.py | 56 +++++++++---------- .../pyproject.toml.tmpl | 5 +- .../cli_tests/test_project_commands.py | 25 +++++++++ 5 files changed, 77 insertions(+), 48 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..aa051604a2b35 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,13 +1,12 @@ import os import sys -from typing import NamedTuple, Optional, Sequence +from typing import NamedTuple, Optional, Sequence, Tuple, Union import click import requests from dagster._generate import ( download_example_from_github, - generate_code_location, generate_project, generate_repository, ) @@ -32,6 +31,7 @@ def project_cli(): ) scaffold_code_location_command_help_text = ( + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster code location, in the current directory. " "This CLI helps you to scaffold a new Dagster code location within a folder structure that " "includes multiple Dagster code locations." @@ -74,6 +74,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) +# start deprecated commands @project_cli.command( name="scaffold-repository", @@ -97,8 +98,7 @@ def scaffold_repository_command(name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`" - " instead.", + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", fg="yellow", ) ) @@ -118,16 +118,16 @@ def scaffold_repository_command(name: str): help="Name of the new Dagster code location", ) def scaffold_code_location_command(name: str): - dir_abspath = os.path.abspath(name) - if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): - click.echo( - click.style(f"The directory {dir_abspath} already exists. ", fg="red") - + "\nPlease delete the contents of this path or choose another location." - ) - sys.exit(1) + scaffold_command(name, excludes="README.md") - generate_code_location(dir_abspath) - click.echo(_styled_success_statement(name, dir_abspath)) + click.echo( + click.style( + "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.", + fg="yellow", + ) + ) + +# end deprecated commands def _check_and_error_on_package_conflicts(project_name: str) -> None: @@ -170,13 +170,21 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: type=click.STRING, help="Name of the new Dagster project", ) +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help="Exclude file patterns from the project template", +) @click.option( "--ignore-package-conflict", is_flag=True, default=False, help="Controls whether the project name can conflict with an existing PyPI package.", ) -def scaffold_command(name: str, ignore_package_conflict: bool): +def scaffold_command(name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool=False): + excludes = list(excludes) dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -188,7 +196,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool): if not ignore_package_conflict: _check_and_error_on_package_conflicts(name) - generate_project(dir_abspath) + generate_project(dir_abspath, excludes) click.echo(_styled_success_statement(name, dir_abspath)) diff --git a/python_modules/dagster/dagster/_generate/__init__.py b/python_modules/dagster/dagster/_generate/__init__.py index 680aa2bc669e5..f6d6b66c15d42 100644 --- a/python_modules/dagster/dagster/_generate/__init__.py +++ b/python_modules/dagster/dagster/_generate/__init__.py @@ -1,6 +1,5 @@ from dagster._generate.download import download_example_from_github as download_example_from_github from dagster._generate.generate import ( - generate_code_location as generate_code_location, generate_project as generate_project, generate_repository as generate_repository, ) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..03c16aa8c3dbc 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os import posixpath @@ -6,7 +8,13 @@ from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"] +IGNORE_PATTERN_LIST : list[str] = [ + "__pycache__", + ".pytest_cache", + "*.egg-info", + ".DS_Store", + "tox.ini", +] def generate_repository(path: str): @@ -14,8 +22,8 @@ def generate_repository(path: str): click.echo(f"Creating a Dagster repository at {path}.") - # Step 1: Generate files for Dagster repository - _generate_files_from_template( + # Render templates for Dagster repository + _render_templates( path=path, name_placeholder=REPO_NAME_PLACEHOLDER, project_template_path=os.path.join( @@ -26,33 +34,13 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): - CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" - - click.echo(f"Creating a Dagster code location at {path}.") - - # Step 1: Generate files for Dagster code location including pyproject.toml, setup.py - _generate_files_from_template( - path=path, - name_placeholder=CODE_LOCATION_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER - ), - ) - - click.echo(f"Generated files for Dagster code location in {path}.") - - def generate_project(path: str): PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" click.echo(f"Creating a Dagster project at {path}.") - # Step 1: Generate files for Dagster code location - generate_code_location(path) - - # Step 2: Generate project-level files, e.g. README - _generate_files_from_template( + # Step 1: Render templates for Dagster project + _render_templates( path=path, name_placeholder=PROJECT_NAME_PLACEHOLDER, project_template_path=os.path.join( @@ -64,8 +52,12 @@ def generate_project(path: str): click.echo(f"Generated files for Dagster project in {path}.") -def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False +def _render_templates( + path: str, + name_placeholder: str, + project_template_path: str, + skip_mkdir: bool = False, + excludes: list[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -76,11 +68,13 @@ def _generate_files_from_template( loader = jinja2.FileSystemLoader(searchpath=project_template_path) env = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list + excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + if _should_skip_file(src_dir_path, excludes): continue src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) @@ -96,7 +90,7 @@ def _generate_files_from_template( # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue src_relative_file_path = os.path.relpath(src_file_path, project_template_path) @@ -124,13 +118,13 @@ def _generate_files_from_template( f.write("\n") -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: + for pattern in excludes: if pattern in path: return True diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 84e7d57e93b61..89c66a746bcfb 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,6 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] + [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 7a0fbf9055520..b889bb81ec168 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -51,6 +51,21 @@ def test_project_scaffold_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_project.definitions" +def test_project_scaffold_command_excludes_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + scaffold_command, + ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], + ) + assert result.exit_code == 0 + assert os.path.exists("diet_dagster/pyproject.toml") + assert os.path.exists("diet_dagster/README.md") + assert not os.path.exists("diet_dagster/diet_dagster_tests/") + assert not os.path.exists("diet_dagster/setup.cfg") + assert not os.path.exists("diet_dagster/setup.py") + + def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): runner = CliRunner() with runner.isolated_filesystem(): @@ -76,6 +91,16 @@ def test_scaffold_code_location_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" +def test_scaffold_code_location_deprecation(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) + assert re.match( + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + result.output, + ) + + def test_from_example_command_fails_when_example_not_available(): runner = CliRunner() with runner.isolated_filesystem(): From b9529d8e8f13c49866d5a609206d19b1f51da13b Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 21 Oct 2024 13:18:09 -0400 Subject: [PATCH 2/8] resolve make ruff --- python_modules/dagster/dagster/_cli/project.py | 15 ++++++++------- .../dagster/dagster/_generate/generate.py | 12 +++++++----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index aa051604a2b35..66254bbe3a8d8 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -5,11 +5,7 @@ import click import requests -from dagster._generate import ( - download_example_from_github, - generate_project, - generate_repository, -) +from dagster._generate import download_example_from_github, generate_project, generate_repository from dagster._generate.download import AVAILABLE_EXAMPLES from dagster.version import __version__ as dagster_version @@ -74,8 +70,10 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) + # start deprecated commands + @project_cli.command( name="scaffold-repository", short_help=scaffold_repository_command_help_text, @@ -126,7 +124,8 @@ def scaffold_code_location_command(name: str): fg="yellow", ) ) - + + # end deprecated commands @@ -183,7 +182,9 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: default=False, help="Controls whether the project name can conflict with an existing PyPI package.", ) -def scaffold_command(name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool=False): +def scaffold_command( + name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool = False +): excludes = list(excludes) dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 03c16aa8c3dbc..3fa7b62a8f25e 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,14 +1,13 @@ -from __future__ import annotations - import os import posixpath +from typing import Optional import click import jinja2 from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST : list[str] = [ +IGNORE_PATTERN_LIST: list[str] = [ "__pycache__", ".pytest_cache", "*.egg-info", @@ -16,6 +15,8 @@ "tox.ini", ] +PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" + def generate_repository(path: str): REPO_NAME_PLACEHOLDER = "REPO_NAME_PLACEHOLDER" @@ -34,8 +35,9 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_project(path: str): - PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" +def generate_project(path: str, excludes: Optional[list[str]] = None): + if not excludes: + excludes = [] click.echo(f"Creating a Dagster project at {path}.") From b79815821e0084330aef4237676bc18f869be327 Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 21 Oct 2024 13:27:39 -0400 Subject: [PATCH 3/8] appease pyright --- python_modules/dagster/dagster/_generate/generate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 3fa7b62a8f25e..cb7558db4db87 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,13 +1,13 @@ import os import posixpath -from typing import Optional +from typing import List, Optional import click import jinja2 from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST: list[str] = [ +IGNORE_PATTERN_LIST: List[str] = [ "__pycache__", ".pytest_cache", "*.egg-info", @@ -35,7 +35,7 @@ def generate_repository(path: str): click.echo(f"Generated files for Dagster repository in {path}.") -def generate_project(path: str, excludes: Optional[list[str]] = None): +def generate_project(path: str, excludes: Optional[List[str]] = None): if not excludes: excludes = [] @@ -59,7 +59,7 @@ def _render_templates( name_placeholder: str, project_template_path: str, skip_mkdir: bool = False, - excludes: list[str] = [], + excludes: List[str] = [], ): normalized_path = os.path.normpath(path) code_location_name = os.path.basename(normalized_path).replace("-", "_") @@ -120,7 +120,7 @@ def _render_templates( f.write("\n") -def _should_skip_file(path: str, excludes: list[str] = IGNORE_PATTERN_LIST): +def _should_skip_file(path: str, excludes: List[str] = IGNORE_PATTERN_LIST): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. From c7701a8bced6c81cdc21b2f1ac75207feb8540c6 Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 21 Oct 2024 13:46:57 -0400 Subject: [PATCH 4/8] use context to invoke scaffold command --- python_modules/dagster/dagster/_cli/project.py | 5 +++-- python_modules/dagster/dagster/_generate/download.py | 1 + .../dagster/dagster_tests/cli_tests/test_project_commands.py | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index 66254bbe3a8d8..7031b11f73591 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -115,8 +115,9 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -def scaffold_code_location_command(name: str): - scaffold_command(name, excludes="README.md") +@click.pass_context +def scaffold_code_location_command(context, name: str): + context.invoke(scaffold_command, name=name, excludes=["README.md"]) click.echo( click.style( diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index d51fb3f81a660..9a14877494be1 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -30,6 +30,7 @@ "deploy_ecs", "deploy_k8s", "development_to_production", + "etl_tutorial", "feature_graph_backed_assets", "getting_started_etl_tutorial", "project_analytics", diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index b889bb81ec168..dcd31bb475c54 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -187,8 +187,7 @@ def test_scaffold_repository_deprecation(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) assert re.match( - "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`" - " instead.", + "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", result.output, ) From 20864807860b61b0c3de1b0003262fe5d2cf766a Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 21 Oct 2024 14:03:12 -0400 Subject: [PATCH 5/8] include excludes in generate_project --- python_modules/dagster/dagster/_cli/project.py | 3 +-- python_modules/dagster/dagster/_generate/generate.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index 7031b11f73591..7a67ac7e82616 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -117,14 +117,13 @@ def scaffold_repository_command(name: str): ) @click.pass_context def scaffold_code_location_command(context, name: str): - context.invoke(scaffold_command, name=name, excludes=["README.md"]) - click.echo( click.style( "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.", fg="yellow", ) ) + context.invoke(scaffold_command, name=name, excludes=["README.md"]) # end deprecated commands diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index cb7558db4db87..5d0c4453afa6c 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -49,6 +49,7 @@ def generate_project(path: str, excludes: Optional[List[str]] = None): os.path.dirname(__file__), "templates", PROJECT_NAME_PLACEHOLDER ), skip_mkdir=True, + excludes=excludes, ) click.echo(f"Generated files for Dagster project in {path}.") From a54ac52da9822b4ce1574a0cf6ba1df1e1ffc57d Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Mon, 21 Oct 2024 14:11:30 -0400 Subject: [PATCH 6/8] rm unneeded project list entry --- python_modules/dagster/dagster/_generate/download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index 9a14877494be1..d51fb3f81a660 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -30,7 +30,6 @@ "deploy_ecs", "deploy_k8s", "development_to_production", - "etl_tutorial", "feature_graph_backed_assets", "getting_started_etl_tutorial", "project_analytics", From 4340fa891f0222d0d262d117e53a983fbb9a2ace Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sun, 27 Oct 2024 01:09:08 +0000 Subject: [PATCH 7/8] eat(cli): add `dagster project scaffold --excludes foo` option Fixed flakey tests by refactor `os.path` to pathlib.Path. --- .../dagster/dagster/_cli/project.py | 106 ++++++++------ .../dagster/dagster/_generate/download.py | 6 +- .../dagster/dagster/_generate/generate.py | 101 ++++++------- .../setup.cfg.tmpl | 2 - .../PROJECT_NAME_PLACEHOLDER}/__init__.py | 0 .../PROJECT_NAME_PLACEHOLDER}/assets.py | 0 .../definitions.py.jinja} | 2 +- .../__init__.py | 0 .../test_assets.py.jinja} | 0 .../{README.md => README.md.jinja} | 8 +- .../pyproject.toml.jinja} | 14 +- .../PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja | 2 + .../setup.py.jinja} | 4 +- .../cli_tests/test_project_commands.py | 137 ++++++++++-------- scripts/install_dev_python_modules.py | 35 +++-- 15 files changed, 218 insertions(+), 199 deletions(-) delete mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/__init__.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/assets.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja} (74%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests}/__init__.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja} (100%) rename python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/{README.md => README.md.jinja} (83%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl => PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja} (64%) create mode 100644 python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl => PROJECT_NAME_PLACEHOLDER/setup.py.jinja} (63%) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index 7a67ac7e82616..d7e2a6bdc1db6 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,6 +1,6 @@ import os import sys -from typing import NamedTuple, Optional, Sequence, Tuple, Union +from typing import NamedTuple, Optional, Sequence import click import requests @@ -18,9 +18,10 @@ def project_cli(): # Keywords to flag in package names. When a project name contains one of these keywords, we check # if a conflicting PyPI package exists. FLAGGED_PACKAGE_KEYWORDS = ["dagster", "dbt"] +VALID_EXCLUDES: list[str] = ["readme.md", "setup", "tests"] # all lower case scaffold_repository_command_help_text = ( - "(DEPRECATED; Use `dagster project scaffold-code-location` instead) " + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster repository, in the current directory. " "This CLI helps you to scaffold a new Dagster repository within a folder structure that " "includes multiple Dagster repositories" @@ -71,7 +72,35 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) -# start deprecated commands +def _check_and_error_on_package_conflicts(project_name: str) -> None: + package_check_result: PackageConflictCheckResult = check_if_pypi_package_conflict_exists( + project_name + ) + if package_check_result.request_error_msg: + click.echo( + click.style( + "An error occurred while checking for package conflicts:" + f" {package_check_result.request_error_msg}. \n\nConflicting package names will" + " cause import errors in your project if the existing PyPI package is included" + " as a dependency in your scaffolded project. If desired, this check can be" + " skipped by adding the `--ignore-package-conflict` flag.", + fg="red", + ) + ) + sys.exit(1) + + if package_check_result.conflict_exists: + click.echo( + click.style( + f"The project '{project_name}' conflicts with an existing PyPI package." + " Conflicting package names will cause import errors in your project if the" + " existing PyPI package is included as a dependency in your scaffolded" + " project. Please choose another name, or add the `--ignore-package-conflict`" + " flag to bypass this check.", + fg="yellow", + ) + ) + sys.exit(1) @project_cli.command( @@ -85,7 +114,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC type=click.STRING, help="Name of the new Dagster repository", ) -def scaffold_repository_command(name: str): +def scaffold_repository_command(name: str) -> None: dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -96,7 +125,7 @@ def scaffold_repository_command(name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", fg="yellow", ) ) @@ -119,45 +148,13 @@ def scaffold_repository_command(name: str): def scaffold_code_location_command(context, name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", fg="yellow", ) ) context.invoke(scaffold_command, name=name, excludes=["README.md"]) -# end deprecated commands - - -def _check_and_error_on_package_conflicts(project_name: str) -> None: - package_check_result = check_if_pypi_package_conflict_exists(project_name) - if package_check_result.request_error_msg: - click.echo( - click.style( - "An error occurred while checking for package conflicts:" - f" {package_check_result.request_error_msg}. \n\nConflicting package names will" - " cause import errors in your project if the existing PyPI package is included" - " as a dependency in your scaffolded project. If desired, this check can be" - " skipped by adding the `--ignore-package-conflict` flag.", - fg="red", - ) - ) - sys.exit(1) - - if package_check_result.conflict_exists: - click.echo( - click.style( - f"The project '{project_name}' conflicts with an existing PyPI package." - " Conflicting package names will cause import errors in your project if the" - " existing PyPI package is included as a dependency in your scaffolded" - " project. Please choose another name, or add the `--ignore-package-conflict`" - " flag to bypass this check.", - fg="yellow", - ) - ) - sys.exit(1) - - @project_cli.command( name="scaffold", short_help=scaffold_command_help_text, @@ -174,7 +171,7 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: multiple=True, type=click.STRING, default=[], - help="Exclude file patterns from the project template", + help=f"Exclude case-insensitive file patterns from the project template. Valid patterns: {VALID_EXCLUDES}", ) @click.option( "--ignore-package-conflict", @@ -183,9 +180,8 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: help="Controls whether the project name can conflict with an existing PyPI package.", ) def scaffold_command( - name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool = False -): - excludes = list(excludes) + name: str, excludes: list[str] | tuple | None = None, ignore_package_conflict: bool = False +) -> None: dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -194,10 +190,26 @@ def scaffold_command( ) sys.exit(1) + if isinstance(excludes, tuple): + excludes = list(excludes) + excludes = [] if not excludes else [item for item in excludes] + + invalid_excludes = [item for item in excludes if item.lower() not in VALID_EXCLUDES] + if invalid_excludes: + click.echo( + click.style( + f"The following strings are invalid options for the excludes tag: {invalid_excludes}", + fg="red", + ) + + f"Choose from {VALID_EXCLUDES}. Case-insensitive. " + ) + sys.exit(1) + excludes = [item.lower() for item in excludes if item in VALID_EXCLUDES] + if not ignore_package_conflict: _check_and_error_on_package_conflicts(name) - generate_project(dir_abspath, excludes) + generate_project(dir_abspath, excludes=excludes) click.echo(_styled_success_statement(name, dir_abspath)) @@ -228,7 +240,7 @@ def scaffold_command( default=dagster_version, show_default=True, ) -def from_example_command(name: Optional[str], example: str, version: str): +def from_example_command(name: Optional[str], example: str, version: str) -> None: name = name or example dir_abspath = os.path.abspath(name) + "/" if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): @@ -238,7 +250,7 @@ def from_example_command(name: Optional[str], example: str, version: str): ) sys.exit(1) else: - os.mkdir(dir_abspath) + os.makedirs(dir_abspath, exist_ok=True) download_example_from_github(dir_abspath, example, version) @@ -250,7 +262,7 @@ def from_example_command(name: Optional[str], example: str, version: str): short_help=list_examples_command_help_text, help=list_examples_command_help_text, ) -def from_example_list_command(): +def from_example_list_command() -> None: click.echo("Examples available in `dagster project from-example`:") click.echo(_styled_list_examples_prints(AVAILABLE_EXAMPLES)) @@ -260,7 +272,7 @@ def _styled_list_examples_prints(examples: Sequence[str]) -> str: return "\n".join([f"* {name}" for name in examples]) -def _styled_success_statement(name: str, path: str): +def _styled_success_statement(name: str, path: str) -> None: return ( click.style("Success!", fg="green") + " Created " diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index d51fb3f81a660..0266b2b59b3f4 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -11,11 +11,13 @@ # Examples aren't that can't be downloaded from the dagster project CLI EXAMPLES_TO_IGNORE = [ "deploy_k8s_beta", - "docs_snippets", "docs_beta_snippets", + "docs_snippets", "experimental", - "temp_pins.txt", "use_case_repository", + "pyproject.toml", + "README.md", + "temp_pins.txt", ] # Hardcoded list of available examples. The list is tested against the examples folder in this mono # repo to make sure it's up-to-date. diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 5d0c4453afa6c..232e348292fff 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,17 +1,18 @@ import os import posixpath -from typing import List, Optional +from typing import List import click import jinja2 from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST: List[str] = [ +DEFAULT_EXCLUDES: List[str] = [ "__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", + ".ruff_cache", "tox.ini", ] @@ -23,112 +24,100 @@ def generate_repository(path: str): click.echo(f"Creating a Dagster repository at {path}.") - # Render templates for Dagster repository - _render_templates( + # + generate_project( path=path, + excludes=None, name_placeholder=REPO_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", REPO_NAME_PLACEHOLDER - ), + templates_path=os.path.join(os.path.dirname(__file__), "templates", REPO_NAME_PLACEHOLDER), ) click.echo(f"Generated files for Dagster repository in {path}.") -def generate_project(path: str, excludes: Optional[List[str]] = None): - if not excludes: - excludes = [] +def generate_project( + path: str, + excludes: List[str] | None = None, + name_placeholder: str = PROJECT_NAME_PLACEHOLDER, + templates_path: str = PROJECT_NAME_PLACEHOLDER, +): + """Renders templates for Dagster project.""" + excludes: list[str] = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes click.echo(f"Creating a Dagster project at {path}.") - # Step 1: Render templates for Dagster project - _render_templates( - path=path, - name_placeholder=PROJECT_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", PROJECT_NAME_PLACEHOLDER - ), - skip_mkdir=True, - excludes=excludes, - ) - - click.echo(f"Generated files for Dagster project in {path}.") - - -def _render_templates( - path: str, - name_placeholder: str, - project_template_path: str, - skip_mkdir: bool = False, - excludes: List[str] = [], -): normalized_path = os.path.normpath(path) - code_location_name = os.path.basename(normalized_path).replace("-", "_") + project_name: str = os.path.basename(normalized_path).replace("-", "_") + os.mkdir(normalized_path) - if not skip_mkdir: # skip if the dir is created by previous command - os.mkdir(normalized_path) - - loader = jinja2.FileSystemLoader(searchpath=project_template_path) - env = jinja2.Environment(loader=loader) + project_template_path: str = os.path.join( + os.path.dirname(__file__), "templates", templates_path + ) + loader: jinja2.loaders.FileSystemLoader = jinja2.FileSystemLoader( + searchpath=project_template_path + ) + env: jinja2.environment.Environment = jinja2.Environment(loader=loader) # merge custom skip_files with the default list - excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: - src_dir_path = os.path.join(root, dirname) + src_dir_path: str = os.path.join(root, dirname) if _should_skip_file(src_dir_path, excludes): continue - src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) - dst_relative_dir_path = src_relative_dir_path.replace( + src_relative_dir_path: str = os.path.relpath(src_dir_path, project_template_path) + dst_relative_dir_path: str = src_relative_dir_path.replace( name_placeholder, - code_location_name, + project_name, 1, ) - dst_dir_path = os.path.join(normalized_path, dst_relative_dir_path) + dst_dir_path: str = os.path.join(normalized_path, dst_relative_dir_path) os.mkdir(dst_dir_path) # For each file in the source template, render a file in the destination. for filename in files: - src_file_path = os.path.join(root, filename) + src_file_path: posixpath = os.path.join(root, filename) if _should_skip_file(src_file_path, excludes): continue - src_relative_file_path = os.path.relpath(src_file_path, project_template_path) - dst_relative_file_path = src_relative_file_path.replace( + src_relative_file_path: str = os.path.relpath(src_file_path, project_template_path) + dst_relative_file_path: str = src_relative_file_path.replace( name_placeholder, - code_location_name, + project_name, 1, ) - dst_file_path = os.path.join(normalized_path, dst_relative_file_path) + dst_file_path: str = os.path.join(normalized_path, dst_relative_file_path) - if dst_file_path.endswith(".tmpl"): - dst_file_path = dst_file_path[: -len(".tmpl")] + if dst_file_path.endswith(".jinja"): + dst_file_path = dst_file_path[: -len(".jinja")] with open(dst_file_path, "w", encoding="utf8") as f: # Jinja template names must use the POSIX path separator "/". - template_name = src_relative_file_path.replace(os.sep, posixpath.sep) - template = env.get_template(name=template_name) + template_name: str = src_relative_file_path.replace(os.sep, posixpath.sep) + template: jinja2.environment.Template = env.get_template(name=template_name) f.write( template.render( - repo_name=code_location_name, # deprecated - code_location_name=code_location_name, + repo_name=project_name, # deprecated + code_location_name=project_name, dagster_version=dagster_version, + project_name=project_name, ) ) f.write("\n") + click.echo(f"Generated files for Dagster project in {path}.") + -def _should_skip_file(path: str, excludes: List[str] = IGNORE_PATTERN_LIST): +def _should_skip_file(path: str, excludes: List[str] = DEFAULT_EXCLUDES): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ for pattern in excludes: - if pattern in path: + if pattern.lower() in path.lower(): return True return False diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl deleted file mode 100644 index adfae7c0745fd..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = {{ code_location_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja similarity index 74% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja index 6dd330fcd8ded..1f59c3f583c8c 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja @@ -1,6 +1,6 @@ from dagster import Definitions, load_assets_from_modules -from . import assets # noqa: TID252 +from {{ project_name }} import assets # noqa: TID252 all_assets = load_assets_from_modules([assets]) diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja similarity index 83% rename from python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja index af2c1cc75743e..f1e6dde91d3bd 100644 --- a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja @@ -1,4 +1,4 @@ -# {{ repo_name }} +# {{ project_name }} This is a [Dagster](https://dagster.io/) project scaffolded with [`dagster project scaffold`](https://docs.dagster.io/getting-started/create-new-project). @@ -18,7 +18,7 @@ dagster dev Open http://localhost:3000 with your browser to see the project. -You can start writing assets in `{{ repo_name }}/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. +You can start writing assets in `{{ project_name }}/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. ## Development @@ -28,10 +28,10 @@ You can specify new Python dependencies in `setup.py`. ### Unit testing -Tests are in the `{{ repo_name }}_tests` directory and you can run tests using `pytest`: +Tests are in the `{{ project_name }}_tests` directory and you can run tests using `pytest`: ```bash -pytest {{ repo_name }}_tests +pytest {{ project_name }}_tests ``` ### Schedules and sensors diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja similarity index 64% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja index 89c66a746bcfb..6e1afe2858a3a 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja @@ -1,5 +1,5 @@ [project] -name = "{{ code_location_name }}" +name = "{{ project_name }}" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,9 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" -[tool.setuptools.packages.find] -exclude=["{{ code_location_name }}_tests"] - [tool.dagster] -module_name = "{{ code_location_name }}.definitions" -code_location_name = "{{ code_location_name }}" +module_name = "{{ project_name }}.definitions" +project_name = "{{ project_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ project_name }}_tests"] \ No newline at end of file diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja new file mode 100644 index 0000000000000..dbfb6531ca2fb --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja @@ -0,0 +1,2 @@ +[metadata] +name = {{ project_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja similarity index 63% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja index 043242d6313bb..114e9232e5749 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja @@ -1,8 +1,8 @@ from setuptools import find_packages, setup setup( - name="{{ code_location_name }}", - packages=find_packages(exclude=["{{ code_location_name }}_tests"]), + name="{{ project_name }}", + packages=find_packages(exclude=["{{ project_name }}_tests"]), install_requires=[ "dagster", "dagster-cloud" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index dcd31bb475c54..f7318e1ad2533 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -1,5 +1,6 @@ import os import re +from pathlib import Path from click.testing import CliRunner from dagster import file_relative_path @@ -19,7 +20,8 @@ def test_project_scaffold_command_fails_when_dir_path_exists(): with runner.isolated_filesystem(): os.mkdir("existing_dir") result = runner.invoke(scaffold_command, ["--name", "existing_dir"]) - assert re.match(r"The directory .* already exists", result.output) + assert "The directory" in result.output + assert "already exists" in result.output assert result.exit_code != 0 @@ -39,11 +41,11 @@ def test_project_scaffold_command_succeeds(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_command, ["--name", "my_dagster_project"]) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/my_dagster_project") - assert os.path.exists("my_dagster_project/my_dagster_project_tests") - assert os.path.exists("my_dagster_project/README.md") - assert os.path.exists("my_dagster_project/pyproject.toml") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/my_dagster_project/__init__.py").exists() + assert Path("my_dagster_project/my_dagster_project_tests/__init__.py").exists() + assert Path("my_dagster_project/README.md").exists() + assert Path("my_dagster_project/pyproject.toml").exists() # test target loadable origins = get_origins_from_toml("my_dagster_project/pyproject.toml") @@ -59,46 +61,21 @@ def test_project_scaffold_command_excludes_succeeds(): ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], ) assert result.exit_code == 0 - assert os.path.exists("diet_dagster/pyproject.toml") - assert os.path.exists("diet_dagster/README.md") - assert not os.path.exists("diet_dagster/diet_dagster_tests/") - assert not os.path.exists("diet_dagster/setup.cfg") - assert not os.path.exists("diet_dagster/setup.py") + assert Path("diet_dagster/pyproject.toml").exists() + assert Path("diet_dagster/README.md").exists() + assert not Path("diet_dagster/diet_dagster_tests/__init__.py").exists() + assert not Path("diet_dagster/setup.cfg").exists() + assert not Path("diet_dagster/setup.py").exists() -def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): +def test_project_scaffold_command_excludes_fails_on_required_files(): runner = CliRunner() with runner.isolated_filesystem(): - os.mkdir("existing_dir") - result = runner.invoke(scaffold_code_location_command, ["--name", "existing_dir"]) - assert re.match(r"The directory .* already exists", result.output) - assert result.exit_code != 0 - - -def test_scaffold_code_location_command_succeeds(): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_code"]) - assert result.exit_code == 0 - assert os.path.exists("my_dagster_code") - assert os.path.exists("my_dagster_code/my_dagster_code") - assert os.path.exists("my_dagster_code/my_dagster_code_tests") - assert os.path.exists("my_dagster_code/pyproject.toml") - - # test target loadable - origins = get_origins_from_toml("my_dagster_code/pyproject.toml") - assert len(origins) == 1 - assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" - - -def test_scaffold_code_location_deprecation(): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) - assert re.match( - "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", - result.output, + result = runner.invoke( + scaffold_command, + ["--name", "diet_dagster", "--excludes", "pyproject"], ) + assert result.exit_code != 0 def test_from_example_command_fails_when_example_not_available(): @@ -119,11 +96,11 @@ def test_from_example_command_succeeds(): ["--name", "my_dagster_project", "--example", "assets_dbt_python"], ) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/assets_dbt_python") - assert os.path.exists("my_dagster_project/assets_dbt_python_tests") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/assets_dbt_python").exists() + assert Path("my_dagster_project/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("my_dagster_project/tox.ini") + assert not Path("my_dagster_project/tox.ini").exists() def test_from_example_command_versioned_succeeds(): @@ -141,11 +118,11 @@ def test_from_example_command_versioned_succeeds(): ], ) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/assets_dbt_python") - assert os.path.exists("my_dagster_project/assets_dbt_python_tests") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/assets_dbt_python").exists() + assert Path("my_dagster_project/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("my_dagster_project/tox.ini") + assert not Path("my_dagster_project/tox.ini").exists() def test_from_example_command_default_name(): @@ -156,21 +133,21 @@ def test_from_example_command_default_name(): ["--name", "assets_dbt_python", "--example", "assets_dbt_python"], ) assert result.exit_code == 0 - assert os.path.exists("assets_dbt_python") - assert os.path.exists("assets_dbt_python/assets_dbt_python") - assert os.path.exists("assets_dbt_python/assets_dbt_python_tests") + assert Path("assets_dbt_python").exists() + assert Path("assets_dbt_python/assets_dbt_python").exists() + assert Path("assets_dbt_python/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("assets_dbt_python/tox.ini") + assert not Path("assets_dbt_python/tox.ini").exists() def test_available_examples_in_sync_with_example_folder(): # ensure the list of AVAILABLE_EXAMPLES is in sync with the example folder minus EXAMPLES_TO_IGNORE - example_folder = file_relative_path(__file__, "../../../../examples") + example_folder = Path(file_relative_path(__file__, "../../../../examples")) available_examples_in_folder = [ e for e in os.listdir(example_folder) if ( - os.path.isdir(os.path.join(example_folder, e)) + Path(example_folder / e).exists() and e not in EXAMPLES_TO_IGNORE and not _should_skip_file(e) ) @@ -178,6 +155,46 @@ def test_available_examples_in_sync_with_example_folder(): assert set(available_examples_in_folder) == set(AVAILABLE_EXAMPLES) +##################### +# `dagster project scaffold-code-location` command is deprecated. + + +def test_scaffold_code_location_deprecation(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_project"]) + assert re.match( + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", + result.output.lstrip(), + ) + + +def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + os.mkdir("existing_dir") + result = runner.invoke(scaffold_code_location_command, ["--name", "existing_dir"]) + assert "The directory" in result.output + assert "already exists" in result.output + assert result.exit_code != 0 + + +def test_scaffold_code_location_command_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_code"]) + assert result.exit_code == 0 + assert Path("my_dagster_code").exists() + assert Path("my_dagster_code/my_dagster_code/__init__.py").exists() + assert Path("my_dagster_code/my_dagster_code_tests/__init__.py").exists() + assert Path("my_dagster_code/pyproject.toml").exists() + + # test target loadable + origins = get_origins_from_toml("my_dagster_code/pyproject.toml") + assert len(origins) == 1 + assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" + + # `dagster project scaffold-repository` command is deprecated. # We're keeping the tests below for backcompat. @@ -187,7 +204,7 @@ def test_scaffold_repository_deprecation(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) assert re.match( - "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", result.output, ) @@ -206,10 +223,10 @@ def test_scaffold_repository_command_succeeds(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_repo"]) assert result.exit_code == 0 - assert os.path.exists("my_dagster_repo") - assert os.path.exists("my_dagster_repo/my_dagster_repo") - assert os.path.exists("my_dagster_repo/my_dagster_repo_tests") - assert not os.path.exists("my_dagster_repo/workspace.yaml") + assert Path("my_dagster_repo").exists() + assert Path("my_dagster_repo/my_dagster_repo").exists() + assert Path("my_dagster_repo/my_dagster_repo_tests").exists() + assert not Path("my_dagster_repo/workspace.yaml").exists() def test_versioned_download(): diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index d97c6bff7c7c9..494534db2687c 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -49,16 +49,22 @@ def main( # Supported on all Python versions. editable_target_paths = [ + ".buildkite/dagster-buildkite", + "examples/experimental/dagster-blueprints", + "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", + "integration_tests/python_modules/dagster-k8s-test-infra", + "helm/dagster/schema[test]", + "python_modules/automation", "python_modules/dagster[pyright,ruff,test]", "python_modules/dagster-pipes", "python_modules/dagster-graphql", "python_modules/dagster-test", "python_modules/dagster-webserver", "python_modules/dagit", - "python_modules/automation", "python_modules/libraries/dagster-managed-elements", "python_modules/libraries/dagster-airbyte", "python_modules/libraries/dagster-aws[stubs,test]", + "python_modules/libraries/dagster-azure", "python_modules/libraries/dagster-celery", "python_modules/libraries/dagster-celery-docker", "python_modules/libraries/dagster-dask[yarn,pbs,kube]", @@ -66,6 +72,9 @@ def main( "python_modules/libraries/dagster-datadog", "python_modules/libraries/dagster-datahub", "python_modules/libraries/dagster-dbt", + "python_modules/libraries/dagster-deltalake", + "python_modules/libraries/dagster-deltalake-pandas", + "python_modules/libraries/dagster-deltalake-polars", "python_modules/libraries/dagster-docker", "python_modules/libraries/dagster-gcp", "python_modules/libraries/dagster-gcp-pandas", @@ -77,31 +86,26 @@ def main( "python_modules/libraries/dagster-celery-k8s", "python_modules/libraries/dagster-github", "python_modules/libraries/dagster-mlflow", + "python_modules/libraries/dagster-msteams", "python_modules/libraries/dagster-mysql", "python_modules/libraries/dagster-looker", "python_modules/libraries/dagster-openai", "python_modules/libraries/dagster-pagerduty", "python_modules/libraries/dagster-pandas", + "python_modules/libraries/dagster-pandera", + "python_modules/libraries/dagster-polars[deltalake,gcp,test]", "python_modules/libraries/dagster-papertrail", "python_modules/libraries/dagster-postgres", "python_modules/libraries/dagster-prometheus", "python_modules/libraries/dagster-pyspark", "python_modules/libraries/dagster-shell", "python_modules/libraries/dagster-slack", + "python_modules/libraries/dagster-snowflake", + "python_modules/libraries/dagster-snowflake-pandas", "python_modules/libraries/dagster-spark", "python_modules/libraries/dagster-ssh", "python_modules/libraries/dagster-twilio", "python_modules/libraries/dagstermill", - "integration_tests/python_modules/dagster-k8s-test-infra", - "python_modules/libraries/dagster-azure", - "python_modules/libraries/dagster-msteams", - "python_modules/libraries/dagster-deltalake", - "python_modules/libraries/dagster-deltalake-pandas", - "python_modules/libraries/dagster-deltalake-polars", - "helm/dagster/schema[test]", - ".buildkite/dagster-buildkite", - "examples/experimental/dagster-blueprints", - "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", ] if sys.version_info <= (3, 12): @@ -114,13 +118,8 @@ def main( "python_modules/libraries/dagster-airflow", ] - if sys.version_info > (3, 7): - editable_target_paths += [ - "python_modules/libraries/dagster-pandera", - "python_modules/libraries/dagster-snowflake", - "python_modules/libraries/dagster-snowflake-pandas", - "python_modules/libraries/dagster-polars[deltalake,gcp,test]", - ] + # if sys.version_info > (3, 7): + # editable_target_paths += [] install_targets += list( itertools.chain.from_iterable( From 4f99f5b2f2df941132eb08a5c71e647e6dfcb0ef Mon Sep 17 00:00:00 2001 From: Colton Padden Date: Tue, 29 Oct 2024 12:22:48 -0400 Subject: [PATCH 8/8] fix pyright --- python_modules/dagster/dagster/_cli/project.py | 10 ++++++---- python_modules/dagster/dagster/_generate/generate.py | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index d7e2a6bdc1db6..1fa5a217a3310 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,6 +1,6 @@ import os import sys -from typing import NamedTuple, Optional, Sequence +from typing import List, NamedTuple, Optional, Sequence, Union import click import requests @@ -11,7 +11,7 @@ @click.group(name="project") -def project_cli(): +def project_cli() -> None: """Commands for bootstrapping new Dagster projects and code locations.""" @@ -180,7 +180,9 @@ def scaffold_code_location_command(context, name: str): help="Controls whether the project name can conflict with an existing PyPI package.", ) def scaffold_command( - name: str, excludes: list[str] | tuple | None = None, ignore_package_conflict: bool = False + name: str, + excludes: Optional[Union[List[str], tuple]] = None, + ignore_package_conflict: bool = False, ) -> None: dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): @@ -272,7 +274,7 @@ def _styled_list_examples_prints(examples: Sequence[str]) -> str: return "\n".join([f"* {name}" for name in examples]) -def _styled_success_statement(name: str, path: str) -> None: +def _styled_success_statement(name: str, path: str) -> str: return ( click.style("Success!", fg="green") + " Created " diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 232e348292fff..a3ce06873ea1b 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,6 +1,6 @@ import os import posixpath -from typing import List +from typing import List, Optional import click import jinja2 @@ -37,12 +37,12 @@ def generate_repository(path: str): def generate_project( path: str, - excludes: List[str] | None = None, + excludes: Optional[List[str]] = None, name_placeholder: str = PROJECT_NAME_PLACEHOLDER, templates_path: str = PROJECT_NAME_PLACEHOLDER, ): """Renders templates for Dagster project.""" - excludes: list[str] = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes + excludes = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes click.echo(f"Creating a Dagster project at {path}.") @@ -78,7 +78,7 @@ def generate_project( # For each file in the source template, render a file in the destination. for filename in files: - src_file_path: posixpath = os.path.join(root, filename) + src_file_path = os.path.join(root, filename) if _should_skip_file(src_file_path, excludes): continue