From a0f13e9099185f02f3615f131b9cb8fbfe05e6e1 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sun, 27 Oct 2024 01:09:08 +0000 Subject: [PATCH] feat(cli): add `dagster project scaffold --excludes foo` option Fixed flakey tests by refactor `os.path` to pathlib.Path. --- .../dagster/dagster/_cli/project.py | 113 ++++++++------- .../dagster/dagster/_generate/download.py | 6 +- .../dagster/dagster/_generate/generate.py | 98 ++++++------- .../setup.cfg.tmpl | 2 - .../DEFINITIONS_PATH_PLACEHOLDER}/__init__.py | 0 .../PROJECT_NAME_PLACEHOLDER}/__init__.py | 0 .../PROJECT_NAME_PLACEHOLDER}/assets.py | 0 .../PROJECT_NAME_PLACEHOLDER}/definitions.py | 2 +- .../__init__.py} | 0 .../test_assets.py.jinja | 0 .../{README.md => README.md.jinja} | 8 +- .../pyproject.toml.jinja} | 14 +- .../PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja | 2 + .../setup.py.jinja} | 4 +- .../cli_tests/test_project_commands.py | 133 +++++++++--------- scripts/install_dev_python_modules.py | 48 +++---- 16 files changed, 210 insertions(+), 220 deletions(-) delete mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_NAME_PLACEHOLDER/DEFINITIONS_PATH_PLACEHOLDER}/__init__.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/__init__.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/assets.py (100%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER}/definitions.py (74%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl => PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py} (100%) create mode 100644 python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja rename python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/{README.md => README.md.jinja} (83%) rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl => PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja} (64%) create mode 100644 python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja rename python_modules/dagster/dagster/_generate/templates/{CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl => PROJECT_NAME_PLACEHOLDER/setup.py.jinja} (63%) diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index 7a67ac7e82616..53958f0deb145 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,6 +1,6 @@ import os import sys -from typing import NamedTuple, Optional, Sequence, Tuple, Union +from typing import NamedTuple, Optional, Sequence import click import requests @@ -20,14 +20,14 @@ def project_cli(): FLAGGED_PACKAGE_KEYWORDS = ["dagster", "dbt"] scaffold_repository_command_help_text = ( - "(DEPRECATED; Use `dagster project scaffold-code-location` instead) " + "(DEPRECATED; Use `dagster project scaffold --excludes readme` instead) " "Create a folder structure with a single Dagster repository, in the current directory. " "This CLI helps you to scaffold a new Dagster repository within a folder structure that " "includes multiple Dagster repositories" ) scaffold_code_location_command_help_text = ( - "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " + "(DEPRECATED; Use `dagster project scaffold --excludes readme` instead) " "Create a folder structure with a single Dagster code location, in the current directory. " "This CLI helps you to scaffold a new Dagster code location within a folder structure that " "includes multiple Dagster code locations." @@ -52,7 +52,7 @@ class PackageConflictCheckResult(NamedTuple): conflict_exists: bool = False -def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictCheckResult: +def _get_pypi_package_conflict_result(project_name: str) -> PackageConflictCheckResult: """Checks if the project name contains any flagged keywords. If so, raises a warning if a PyPI package with the same name exists. This is to prevent import errors from occurring due to a project name that conflicts with an imported package. @@ -71,7 +71,35 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) -# start deprecated commands +def check_pypi_package_conflict(project_name: str) -> None: + package_check_result: PackageConflictCheckResult = _get_pypi_package_conflict_result( + project_name + ) + if package_check_result.request_error_msg: + click.echo( + click.style( + "An error occurred while checking for package conflicts:" + f" {package_check_result.request_error_msg}. \n\nConflicting package names will" + " cause import errors in your project if the existing PyPI package is included" + " as a dependency in your scaffolded project. If desired, this check can be" + " skipped by adding the `--ignore-package-conflict` flag.", + fg="red", + ) + ) + sys.exit(1) + + if package_check_result.conflict_exists: + click.echo( + click.style( + f"The project '{project_name}' conflicts with an existing PyPI package." + " Conflicting package names will cause import errors in your project if the" + " existing PyPI package is included as a dependency in your scaffolded" + " project. Please choose another name, or add the `--ignore-package-conflict`" + " flag to bypass this check.", + fg="yellow", + ) + ) + sys.exit(1) @project_cli.command( @@ -85,7 +113,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC type=click.STRING, help="Name of the new Dagster repository", ) -def scaffold_repository_command(name: str): +def scaffold_repository_command(name: str) -> None: dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -96,7 +124,7 @@ def scaffold_repository_command(name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", fg="yellow", ) ) @@ -115,47 +143,23 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -@click.pass_context -def scaffold_code_location_command(context, name: str): - click.echo( - click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold --excludes README.md` instead.", - fg="yellow", - ) - ) - context.invoke(scaffold_command, name=name, excludes=["README.md"]) - - -# end deprecated commands - - -def _check_and_error_on_package_conflicts(project_name: str) -> None: - package_check_result = check_if_pypi_package_conflict_exists(project_name) - if package_check_result.request_error_msg: +def scaffold_code_location_command(name: str) -> None: + dir_abspath = os.path.abspath(name) + if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( - click.style( - "An error occurred while checking for package conflicts:" - f" {package_check_result.request_error_msg}. \n\nConflicting package names will" - " cause import errors in your project if the existing PyPI package is included" - " as a dependency in your scaffolded project. If desired, this check can be" - " skipped by adding the `--ignore-package-conflict` flag.", - fg="red", - ) + click.style(f"The directory {dir_abspath} already exists. ", fg="red") + + "\nPlease delete the contents of this path or choose another location." ) sys.exit(1) - if package_check_result.conflict_exists: - click.echo( - click.style( - f"The project '{project_name}' conflicts with an existing PyPI package." - " Conflicting package names will cause import errors in your project if the" - " existing PyPI package is included as a dependency in your scaffolded" - " project. Please choose another name, or add the `--ignore-package-conflict`" - " flag to bypass this check.", - fg="yellow", - ) + click.echo( + click.style( + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", + fg="yellow", ) - sys.exit(1) + ) + generate_project(dir_abspath, excludes=["README"]) + click.echo(_styled_success_statement(name, dir_abspath)) @project_cli.command( @@ -174,7 +178,7 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: multiple=True, type=click.STRING, default=[], - help="Exclude file patterns from the project template", + help="Exclude case-insensitive file patterns from the project template", ) @click.option( "--ignore-package-conflict", @@ -183,9 +187,12 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: help="Controls whether the project name can conflict with an existing PyPI package.", ) def scaffold_command( - name: str, excludes: Union[Tuple, list], ignore_package_conflict: bool = False -): - excludes = list(excludes) + name: str, excludes: list[str] | tuple | None = None, ignore_package_conflict: bool = False +) -> None: + excludes = [] if not excludes else excludes + if isinstance(excludes, tuple): + excludes = list(excludes) + dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -195,9 +202,9 @@ def scaffold_command( sys.exit(1) if not ignore_package_conflict: - _check_and_error_on_package_conflicts(name) + check_pypi_package_conflict(name) - generate_project(dir_abspath, excludes) + generate_project(dir_abspath, excludes=excludes) click.echo(_styled_success_statement(name, dir_abspath)) @@ -228,7 +235,7 @@ def scaffold_command( default=dagster_version, show_default=True, ) -def from_example_command(name: Optional[str], example: str, version: str): +def from_example_command(name: Optional[str], example: str, version: str) -> None: name = name or example dir_abspath = os.path.abspath(name) + "/" if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): @@ -238,7 +245,7 @@ def from_example_command(name: Optional[str], example: str, version: str): ) sys.exit(1) else: - os.mkdir(dir_abspath) + os.makedirs(dir_abspath, exist_ok=True) download_example_from_github(dir_abspath, example, version) @@ -250,7 +257,7 @@ def from_example_command(name: Optional[str], example: str, version: str): short_help=list_examples_command_help_text, help=list_examples_command_help_text, ) -def from_example_list_command(): +def from_example_list_command() -> None: click.echo("Examples available in `dagster project from-example`:") click.echo(_styled_list_examples_prints(AVAILABLE_EXAMPLES)) @@ -260,7 +267,7 @@ def _styled_list_examples_prints(examples: Sequence[str]) -> str: return "\n".join([f"* {name}" for name in examples]) -def _styled_success_statement(name: str, path: str): +def _styled_success_statement(name: str, path: str) -> None: return ( click.style("Success!", fg="green") + " Created " diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index d51fb3f81a660..0266b2b59b3f4 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -11,11 +11,13 @@ # Examples aren't that can't be downloaded from the dagster project CLI EXAMPLES_TO_IGNORE = [ "deploy_k8s_beta", - "docs_snippets", "docs_beta_snippets", + "docs_snippets", "experimental", - "temp_pins.txt", "use_case_repository", + "pyproject.toml", + "README.md", + "temp_pins.txt", ] # Hardcoded list of available examples. The list is tested against the examples folder in this mono # repo to make sure it's up-to-date. diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 5d0c4453afa6c..a382ddec3a496 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,17 +1,18 @@ import os import posixpath -from typing import List, Optional +from typing import List import click import jinja2 from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST: List[str] = [ +DEFAULT_EXCLUDES: List[str] = [ "__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", + ".ruff_cache", "tox.ini", ] @@ -24,111 +25,98 @@ def generate_repository(path: str): click.echo(f"Creating a Dagster repository at {path}.") # Render templates for Dagster repository - _render_templates( + generate_project( path=path, + excludes=None, name_placeholder=REPO_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", REPO_NAME_PLACEHOLDER - ), + templates_path=os.path.join(os.path.dirname(__file__), "templates", REPO_NAME_PLACEHOLDER), ) click.echo(f"Generated files for Dagster repository in {path}.") -def generate_project(path: str, excludes: Optional[List[str]] = None): - if not excludes: - excludes = [] +def generate_project( + path: str, + excludes: List[str] | None = None, + name_placeholder: str = PROJECT_NAME_PLACEHOLDER, + templates_path: str = PROJECT_NAME_PLACEHOLDER, +): + excludes: list[str] = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes click.echo(f"Creating a Dagster project at {path}.") - # Step 1: Render templates for Dagster project - _render_templates( - path=path, - name_placeholder=PROJECT_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", PROJECT_NAME_PLACEHOLDER - ), - skip_mkdir=True, - excludes=excludes, - ) - - click.echo(f"Generated files for Dagster project in {path}.") - - -def _render_templates( - path: str, - name_placeholder: str, - project_template_path: str, - skip_mkdir: bool = False, - excludes: List[str] = [], -): normalized_path = os.path.normpath(path) - code_location_name = os.path.basename(normalized_path).replace("-", "_") + project_name: str = os.path.basename(normalized_path).replace("-", "_") + os.mkdir(normalized_path) - if not skip_mkdir: # skip if the dir is created by previous command - os.mkdir(normalized_path) - - loader = jinja2.FileSystemLoader(searchpath=project_template_path) - env = jinja2.Environment(loader=loader) + project_template_path: str = os.path.join( + os.path.dirname(__file__), "templates", templates_path + ) + loader: jinja2.loaders.FileSystemLoader = jinja2.FileSystemLoader( + searchpath=project_template_path + ) + env: jinja2.environment.Environment = jinja2.Environment(loader=loader) # merge custom skip_files with the default list - excludes = IGNORE_PATTERN_LIST + excludes for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: - src_dir_path = os.path.join(root, dirname) + src_dir_path: str = os.path.join(root, dirname) if _should_skip_file(src_dir_path, excludes): continue - src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) - dst_relative_dir_path = src_relative_dir_path.replace( + src_relative_dir_path: str = os.path.relpath(src_dir_path, project_template_path) + dst_relative_dir_path: str = src_relative_dir_path.replace( name_placeholder, - code_location_name, + project_name, 1, ) - dst_dir_path = os.path.join(normalized_path, dst_relative_dir_path) + dst_dir_path: str = os.path.join(normalized_path, dst_relative_dir_path) os.mkdir(dst_dir_path) # For each file in the source template, render a file in the destination. for filename in files: - src_file_path = os.path.join(root, filename) + src_file_path: posixpath = os.path.join(root, filename) if _should_skip_file(src_file_path, excludes): continue - src_relative_file_path = os.path.relpath(src_file_path, project_template_path) - dst_relative_file_path = src_relative_file_path.replace( + src_relative_file_path: str = os.path.relpath(src_file_path, project_template_path) + dst_relative_file_path: str = src_relative_file_path.replace( name_placeholder, - code_location_name, + project_name, 1, ) - dst_file_path = os.path.join(normalized_path, dst_relative_file_path) + dst_file_path: str = os.path.join(normalized_path, dst_relative_file_path) - if dst_file_path.endswith(".tmpl"): - dst_file_path = dst_file_path[: -len(".tmpl")] + if dst_file_path.endswith(".jinja"): + dst_file_path = dst_file_path[: -len(".jinja")] with open(dst_file_path, "w", encoding="utf8") as f: # Jinja template names must use the POSIX path separator "/". - template_name = src_relative_file_path.replace(os.sep, posixpath.sep) - template = env.get_template(name=template_name) + template_name: str = src_relative_file_path.replace(os.sep, posixpath.sep) + template: jinja2.environment.Template = env.get_template(name=template_name) f.write( template.render( - repo_name=code_location_name, # deprecated - code_location_name=code_location_name, + repo_name=project_name, # deprecated + code_location_name=project_name, dagster_version=dagster_version, + project_name=project_name, ) ) f.write("\n") + click.echo(f"Generated files for Dagster project in {path}.") + -def _should_skip_file(path: str, excludes: List[str] = IGNORE_PATTERN_LIST): +def _should_skip_file(path: str, excludes: List[str] = DEFAULT_EXCLUDES): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ for pattern in excludes: - if pattern in path: + if pattern.lower() in path.lower(): return True return False diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl deleted file mode 100644 index adfae7c0745fd..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = {{ code_location_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/DEFINITIONS_PATH_PLACEHOLDER/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/DEFINITIONS_PATH_PLACEHOLDER/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py similarity index 74% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py index 6dd330fcd8ded..1f59c3f583c8c 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py @@ -1,6 +1,6 @@ from dagster import Definitions, load_assets_from_modules -from . import assets # noqa: TID252 +from {{ project_name }} import assets # noqa: TID252 all_assets = load_assets_from_modules([assets]) diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja similarity index 83% rename from python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja index af2c1cc75743e..f1e6dde91d3bd 100644 --- a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja @@ -1,4 +1,4 @@ -# {{ repo_name }} +# {{ project_name }} This is a [Dagster](https://dagster.io/) project scaffolded with [`dagster project scaffold`](https://docs.dagster.io/getting-started/create-new-project). @@ -18,7 +18,7 @@ dagster dev Open http://localhost:3000 with your browser to see the project. -You can start writing assets in `{{ repo_name }}/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. +You can start writing assets in `{{ project_name }}/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. ## Development @@ -28,10 +28,10 @@ You can specify new Python dependencies in `setup.py`. ### Unit testing -Tests are in the `{{ repo_name }}_tests` directory and you can run tests using `pytest`: +Tests are in the `{{ project_name }}_tests` directory and you can run tests using `pytest`: ```bash -pytest {{ repo_name }}_tests +pytest {{ project_name }}_tests ``` ### Schedules and sensors diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja similarity index 64% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja index 6b483271965cd..551fd07302be4 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja @@ -1,5 +1,5 @@ [project] -name = "{{ code_location_name }}" +name = "{{ project_name }}" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -11,7 +11,7 @@ dependencies = [ [project.optional-dependencies] dev = [ - "dagster-webserver", + "dagster-webserver", "pytest", ] @@ -19,9 +19,9 @@ dev = [ requires = ["setuptools"] build-backend = "setuptools.build_meta" -[tool.setuptools.packages.find] -exclude=["{{ code_location_name }}_tests"] - [tool.dagster] -module_name = "{{ code_location_name }}.definitions" -code_location_name = "{{ code_location_name }}" +module_name = "{{ project_name }}.definitions" +project_name = "{{ project_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ project_name }}_tests"] \ No newline at end of file diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja new file mode 100644 index 0000000000000..dbfb6531ca2fb --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja @@ -0,0 +1,2 @@ +[metadata] +name = {{ project_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja similarity index 63% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja index 043242d6313bb..114e9232e5749 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja @@ -1,8 +1,8 @@ from setuptools import find_packages, setup setup( - name="{{ code_location_name }}", - packages=find_packages(exclude=["{{ code_location_name }}_tests"]), + name="{{ project_name }}", + packages=find_packages(exclude=["{{ project_name }}_tests"]), install_requires=[ "dagster", "dagster-cloud" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index dcd31bb475c54..59a78ea628776 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -1,5 +1,6 @@ import os import re +from pathlib import Path from click.testing import CliRunner from dagster import file_relative_path @@ -39,11 +40,11 @@ def test_project_scaffold_command_succeeds(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_command, ["--name", "my_dagster_project"]) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/my_dagster_project") - assert os.path.exists("my_dagster_project/my_dagster_project_tests") - assert os.path.exists("my_dagster_project/README.md") - assert os.path.exists("my_dagster_project/pyproject.toml") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/my_dagster_project/__init__.py").exists() + assert Path("my_dagster_project/my_dagster_project_tests/__init__.py").exists() + assert Path("my_dagster_project/README.md").exists() + assert Path("my_dagster_project/pyproject.toml").exists() # test target loadable origins = get_origins_from_toml("my_dagster_project/pyproject.toml") @@ -59,46 +60,11 @@ def test_project_scaffold_command_excludes_succeeds(): ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], ) assert result.exit_code == 0 - assert os.path.exists("diet_dagster/pyproject.toml") - assert os.path.exists("diet_dagster/README.md") - assert not os.path.exists("diet_dagster/diet_dagster_tests/") - assert not os.path.exists("diet_dagster/setup.cfg") - assert not os.path.exists("diet_dagster/setup.py") - - -def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): - runner = CliRunner() - with runner.isolated_filesystem(): - os.mkdir("existing_dir") - result = runner.invoke(scaffold_code_location_command, ["--name", "existing_dir"]) - assert re.match(r"The directory .* already exists", result.output) - assert result.exit_code != 0 - - -def test_scaffold_code_location_command_succeeds(): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_code"]) - assert result.exit_code == 0 - assert os.path.exists("my_dagster_code") - assert os.path.exists("my_dagster_code/my_dagster_code") - assert os.path.exists("my_dagster_code/my_dagster_code_tests") - assert os.path.exists("my_dagster_code/pyproject.toml") - - # test target loadable - origins = get_origins_from_toml("my_dagster_code/pyproject.toml") - assert len(origins) == 1 - assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" - - -def test_scaffold_code_location_deprecation(): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) - assert re.match( - "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", - result.output, - ) + assert Path("diet_dagster/pyproject.toml").exists() + assert Path("diet_dagster/README.md").exists() + assert not Path("diet_dagster/diet_dagster_tests/__init__.py").exists() + assert not Path("diet_dagster/setup.cfg").exists() + assert not Path("diet_dagster/setup.py").exists() def test_from_example_command_fails_when_example_not_available(): @@ -119,11 +85,11 @@ def test_from_example_command_succeeds(): ["--name", "my_dagster_project", "--example", "assets_dbt_python"], ) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/assets_dbt_python") - assert os.path.exists("my_dagster_project/assets_dbt_python_tests") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/assets_dbt_python").exists() + assert Path("my_dagster_project/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("my_dagster_project/tox.ini") + assert not Path("my_dagster_project/tox.ini").exists() def test_from_example_command_versioned_succeeds(): @@ -141,11 +107,11 @@ def test_from_example_command_versioned_succeeds(): ], ) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/assets_dbt_python") - assert os.path.exists("my_dagster_project/assets_dbt_python_tests") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/assets_dbt_python").exists() + assert Path("my_dagster_project/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("my_dagster_project/tox.ini") + assert not Path("my_dagster_project/tox.ini").exists() def test_from_example_command_default_name(): @@ -156,21 +122,21 @@ def test_from_example_command_default_name(): ["--name", "assets_dbt_python", "--example", "assets_dbt_python"], ) assert result.exit_code == 0 - assert os.path.exists("assets_dbt_python") - assert os.path.exists("assets_dbt_python/assets_dbt_python") - assert os.path.exists("assets_dbt_python/assets_dbt_python_tests") + assert Path("assets_dbt_python").exists() + assert Path("assets_dbt_python/assets_dbt_python").exists() + assert Path("assets_dbt_python/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("assets_dbt_python/tox.ini") + assert not Path("assets_dbt_python/tox.ini").exists() def test_available_examples_in_sync_with_example_folder(): # ensure the list of AVAILABLE_EXAMPLES is in sync with the example folder minus EXAMPLES_TO_IGNORE - example_folder = file_relative_path(__file__, "../../../../examples") + example_folder = Path(file_relative_path(__file__, "../../../../examples")) available_examples_in_folder = [ e for e in os.listdir(example_folder) if ( - os.path.isdir(os.path.join(example_folder, e)) + Path(example_folder / e).exists() and e not in EXAMPLES_TO_IGNORE and not _should_skip_file(e) ) @@ -178,6 +144,45 @@ def test_available_examples_in_sync_with_example_folder(): assert set(available_examples_in_folder) == set(AVAILABLE_EXAMPLES) +##################### +# `dagster project scaffold-code-location` command is deprecated. + + +def test_scaffold_code_location_deprecation(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_project"]) + assert re.match( + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", + result.output.lstrip(), + ) + + +def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + os.mkdir("existing_dir") + result = runner.invoke(scaffold_code_location_command, ["--name", "existing_dir"]) + assert re.match(r"The directory .* already exists", result.output) + assert result.exit_code != 0 + + +def test_scaffold_code_location_command_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_code"]) + assert result.exit_code == 0 + assert Path("my_dagster_code").exists() + assert Path("my_dagster_code/my_dagster_code/__init__.py").exists() + assert Path("my_dagster_code/my_dagster_code_tests/__init__.py").exists() + assert Path("my_dagster_code/pyproject.toml").exists() + + # test target loadable + origins = get_origins_from_toml("my_dagster_code/pyproject.toml") + assert len(origins) == 1 + assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" + + # `dagster project scaffold-repository` command is deprecated. # We're keeping the tests below for backcompat. @@ -187,7 +192,7 @@ def test_scaffold_repository_deprecation(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) assert re.match( - "WARNING: This command is deprecated. Use `dagster project scaffold` instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", result.output, ) @@ -206,10 +211,10 @@ def test_scaffold_repository_command_succeeds(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_repo"]) assert result.exit_code == 0 - assert os.path.exists("my_dagster_repo") - assert os.path.exists("my_dagster_repo/my_dagster_repo") - assert os.path.exists("my_dagster_repo/my_dagster_repo_tests") - assert not os.path.exists("my_dagster_repo/workspace.yaml") + assert Path("my_dagster_repo").exists() + assert Path("my_dagster_repo/my_dagster_repo").exists() + assert Path("my_dagster_repo/my_dagster_repo_tests").exists() + assert not Path("my_dagster_repo/workspace.yaml").exists() def test_versioned_download(): diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index af860e611b140..3be3dee736c3b 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -49,16 +49,22 @@ def main( # Supported on all Python versions. editable_target_paths = [ + ".buildkite/dagster-buildkite", + "examples/experimental/dagster-blueprints", + "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", + "integration_tests/python_modules/dagster-k8s-test-infra", + "helm/dagster/schema[test]", + "python_modules/automation", "python_modules/dagster[pyright,ruff,test]", "python_modules/dagster-pipes", "python_modules/dagster-graphql", "python_modules/dagster-test", "python_modules/dagster-webserver", "python_modules/dagit", - "python_modules/automation", "python_modules/libraries/dagster-managed-elements", "python_modules/libraries/dagster-airbyte", "python_modules/libraries/dagster-aws[stubs,test]", + "python_modules/libraries/dagster-azure", "python_modules/libraries/dagster-celery", "python_modules/libraries/dagster-celery-docker", "python_modules/libraries/dagster-dask[yarn,pbs,kube]", @@ -66,6 +72,9 @@ def main( "python_modules/libraries/dagster-datadog", "python_modules/libraries/dagster-datahub", "python_modules/libraries/dagster-dbt", + "python_modules/libraries/dagster-deltalake", + "python_modules/libraries/dagster-deltalake-pandas", + "python_modules/libraries/dagster-deltalake-polars", "python_modules/libraries/dagster-docker", "python_modules/libraries/dagster-gcp", "python_modules/libraries/dagster-gcp-pandas", @@ -76,31 +85,26 @@ def main( "python_modules/libraries/dagster-celery-k8s", "python_modules/libraries/dagster-github", "python_modules/libraries/dagster-mlflow", + "python_modules/libraries/dagster-msteams", "python_modules/libraries/dagster-mysql", "python_modules/libraries/dagster-looker", "python_modules/libraries/dagster-openai", "python_modules/libraries/dagster-pagerduty", "python_modules/libraries/dagster-pandas", + "python_modules/libraries/dagster-pandera", + "python_modules/libraries/dagster-polars[deltalake,gcp,test]", "python_modules/libraries/dagster-papertrail", "python_modules/libraries/dagster-postgres", "python_modules/libraries/dagster-prometheus", "python_modules/libraries/dagster-pyspark", "python_modules/libraries/dagster-shell", "python_modules/libraries/dagster-slack", + "python_modules/libraries/dagster-snowflake", + "python_modules/libraries/dagster-snowflake-pandas", "python_modules/libraries/dagster-spark", "python_modules/libraries/dagster-ssh", "python_modules/libraries/dagster-twilio", - "python_modules/libraries/dagstermill", - "integration_tests/python_modules/dagster-k8s-test-infra", - "python_modules/libraries/dagster-azure", - "python_modules/libraries/dagster-msteams", - "python_modules/libraries/dagster-deltalake", - "python_modules/libraries/dagster-deltalake-pandas", - "python_modules/libraries/dagster-deltalake-polars", - "helm/dagster/schema[test]", - ".buildkite/dagster-buildkite", - "examples/experimental/dagster-blueprints", - "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", + "python_modules/libraries/dagstermill", ] if sys.version_info <= (3, 12): @@ -113,13 +117,8 @@ def main( "python_modules/libraries/dagster-airflow", ] - if sys.version_info > (3, 7): - editable_target_paths += [ - "python_modules/libraries/dagster-pandera", - "python_modules/libraries/dagster-snowflake", - "python_modules/libraries/dagster-snowflake-pandas", - "python_modules/libraries/dagster-polars[deltalake,gcp,test]", - ] + # if sys.version_info > (3, 7): + # editable_target_paths += [] install_targets += list( itertools.chain.from_iterable( @@ -136,17 +135,6 @@ def main( "https://github.com/dagster-io/build-grpcio/wiki/Wheels", ] - # NOTE: `dagster-ge` is out of date and does not support recent versions of great expectations. - # Because of this, it has second-order dependencies on old versions of popular libraries like - # numpy which conflict with the requirements of our other libraries. For this reason, until - # dagster-ge is updated we won't install `dagster-ge` in the common dev environment or - # pre-install its dependencies in our BK images (which this script is used for). - # - # dagster-ge depends on a great_expectations version that does not install on Windows - # https://github.com/dagster-io/dagster/issues/3319 - # if sys.version_info >= (3, 7) and os.name != "nt": - # install_targets += ["-e python_modules/libraries/dagster-ge"] - # Ensure uv is installed which we use for faster package resolution subprocess.run(["pip", "install", "-U", "uv"], check=True)