diff --git a/python_modules/dagster/dagster/_cli/project.py b/python_modules/dagster/dagster/_cli/project.py index cb37c3f555151..1fa5a217a3310 100644 --- a/python_modules/dagster/dagster/_cli/project.py +++ b/python_modules/dagster/dagster/_cli/project.py @@ -1,37 +1,34 @@ import os import sys -from typing import NamedTuple, Optional, Sequence +from typing import List, NamedTuple, Optional, Sequence, Union import click import requests -from dagster._generate import ( - download_example_from_github, - generate_code_location, - generate_project, - generate_repository, -) +from dagster._generate import download_example_from_github, generate_project, generate_repository from dagster._generate.download import AVAILABLE_EXAMPLES from dagster.version import __version__ as dagster_version @click.group(name="project") -def project_cli(): +def project_cli() -> None: """Commands for bootstrapping new Dagster projects and code locations.""" # Keywords to flag in package names. When a project name contains one of these keywords, we check # if a conflicting PyPI package exists. FLAGGED_PACKAGE_KEYWORDS = ["dagster", "dbt"] +VALID_EXCLUDES: list[str] = ["readme.md", "setup", "tests"] # all lower case scaffold_repository_command_help_text = ( - "(DEPRECATED; Use `dagster project scaffold-code-location` instead) " + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster repository, in the current directory. " "This CLI helps you to scaffold a new Dagster repository within a folder structure that " "includes multiple Dagster repositories" ) scaffold_code_location_command_help_text = ( + "(DEPRECATED; Use `dagster project scaffold --excludes README.md` instead) " "Create a folder structure with a single Dagster code location, in the current directory. " "This CLI helps you to scaffold a new Dagster code location within a folder structure that " "includes multiple Dagster code locations." @@ -75,6 +72,37 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC return PackageConflictCheckResult(request_error_msg=None, conflict_exists=False) +def _check_and_error_on_package_conflicts(project_name: str) -> None: + package_check_result: PackageConflictCheckResult = check_if_pypi_package_conflict_exists( + project_name + ) + if package_check_result.request_error_msg: + click.echo( + click.style( + "An error occurred while checking for package conflicts:" + f" {package_check_result.request_error_msg}. \n\nConflicting package names will" + " cause import errors in your project if the existing PyPI package is included" + " as a dependency in your scaffolded project. If desired, this check can be" + " skipped by adding the `--ignore-package-conflict` flag.", + fg="red", + ) + ) + sys.exit(1) + + if package_check_result.conflict_exists: + click.echo( + click.style( + f"The project '{project_name}' conflicts with an existing PyPI package." + " Conflicting package names will cause import errors in your project if the" + " existing PyPI package is included as a dependency in your scaffolded" + " project. Please choose another name, or add the `--ignore-package-conflict`" + " flag to bypass this check.", + fg="yellow", + ) + ) + sys.exit(1) + + @project_cli.command( name="scaffold-repository", short_help=scaffold_repository_command_help_text, @@ -86,7 +114,7 @@ def check_if_pypi_package_conflict_exists(project_name: str) -> PackageConflictC type=click.STRING, help="Name of the new Dagster repository", ) -def scaffold_repository_command(name: str): +def scaffold_repository_command(name: str) -> None: dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -97,8 +125,7 @@ def scaffold_repository_command(name: str): click.echo( click.style( - "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`" - " instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", fg="yellow", ) ) @@ -117,46 +144,15 @@ def scaffold_repository_command(name: str): type=click.STRING, help="Name of the new Dagster code location", ) -def scaffold_code_location_command(name: str): - dir_abspath = os.path.abspath(name) - if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): - click.echo( - click.style(f"The directory {dir_abspath} already exists. ", fg="red") - + "\nPlease delete the contents of this path or choose another location." - ) - sys.exit(1) - - generate_code_location(dir_abspath) - click.echo(_styled_success_statement(name, dir_abspath)) - - -def _check_and_error_on_package_conflicts(project_name: str) -> None: - package_check_result = check_if_pypi_package_conflict_exists(project_name) - if package_check_result.request_error_msg: - click.echo( - click.style( - "An error occurred while checking for package conflicts:" - f" {package_check_result.request_error_msg}. \n\nConflicting package names will" - " cause import errors in your project if the existing PyPI package is included" - " as a dependency in your scaffolded project. If desired, this check can be" - " skipped by adding the `--ignore-package-conflict` flag.", - fg="red", - ) - ) - sys.exit(1) - - if package_check_result.conflict_exists: - click.echo( - click.style( - f"The project '{project_name}' conflicts with an existing PyPI package." - " Conflicting package names will cause import errors in your project if the" - " existing PyPI package is included as a dependency in your scaffolded" - " project. Please choose another name, or add the `--ignore-package-conflict`" - " flag to bypass this check.", - fg="yellow", - ) +@click.pass_context +def scaffold_code_location_command(context, name: str): + click.echo( + click.style( + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", + fg="yellow", ) - sys.exit(1) + ) + context.invoke(scaffold_command, name=name, excludes=["README.md"]) @project_cli.command( @@ -170,13 +166,24 @@ def _check_and_error_on_package_conflicts(project_name: str) -> None: type=click.STRING, help="Name of the new Dagster project", ) +@click.option( + "--excludes", + multiple=True, + type=click.STRING, + default=[], + help=f"Exclude case-insensitive file patterns from the project template. Valid patterns: {VALID_EXCLUDES}", +) @click.option( "--ignore-package-conflict", is_flag=True, default=False, help="Controls whether the project name can conflict with an existing PyPI package.", ) -def scaffold_command(name: str, ignore_package_conflict: bool): +def scaffold_command( + name: str, + excludes: Optional[Union[List[str], tuple]] = None, + ignore_package_conflict: bool = False, +) -> None: dir_abspath = os.path.abspath(name) if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): click.echo( @@ -185,10 +192,26 @@ def scaffold_command(name: str, ignore_package_conflict: bool): ) sys.exit(1) + if isinstance(excludes, tuple): + excludes = list(excludes) + excludes = [] if not excludes else [item for item in excludes] + + invalid_excludes = [item for item in excludes if item.lower() not in VALID_EXCLUDES] + if invalid_excludes: + click.echo( + click.style( + f"The following strings are invalid options for the excludes tag: {invalid_excludes}", + fg="red", + ) + + f"Choose from {VALID_EXCLUDES}. Case-insensitive. " + ) + sys.exit(1) + excludes = [item.lower() for item in excludes if item in VALID_EXCLUDES] + if not ignore_package_conflict: _check_and_error_on_package_conflicts(name) - generate_project(dir_abspath) + generate_project(dir_abspath, excludes=excludes) click.echo(_styled_success_statement(name, dir_abspath)) @@ -219,7 +242,7 @@ def scaffold_command(name: str, ignore_package_conflict: bool): default=dagster_version, show_default=True, ) -def from_example_command(name: Optional[str], example: str, version: str): +def from_example_command(name: Optional[str], example: str, version: str) -> None: name = name or example dir_abspath = os.path.abspath(name) + "/" if os.path.isdir(dir_abspath) and os.path.exists(dir_abspath): @@ -229,7 +252,7 @@ def from_example_command(name: Optional[str], example: str, version: str): ) sys.exit(1) else: - os.mkdir(dir_abspath) + os.makedirs(dir_abspath, exist_ok=True) download_example_from_github(dir_abspath, example, version) @@ -241,7 +264,7 @@ def from_example_command(name: Optional[str], example: str, version: str): short_help=list_examples_command_help_text, help=list_examples_command_help_text, ) -def from_example_list_command(): +def from_example_list_command() -> None: click.echo("Examples available in `dagster project from-example`:") click.echo(_styled_list_examples_prints(AVAILABLE_EXAMPLES)) @@ -251,7 +274,7 @@ def _styled_list_examples_prints(examples: Sequence[str]) -> str: return "\n".join([f"* {name}" for name in examples]) -def _styled_success_statement(name: str, path: str): +def _styled_success_statement(name: str, path: str) -> str: return ( click.style("Success!", fg="green") + " Created " diff --git a/python_modules/dagster/dagster/_generate/__init__.py b/python_modules/dagster/dagster/_generate/__init__.py index 680aa2bc669e5..f6d6b66c15d42 100644 --- a/python_modules/dagster/dagster/_generate/__init__.py +++ b/python_modules/dagster/dagster/_generate/__init__.py @@ -1,6 +1,5 @@ from dagster._generate.download import download_example_from_github as download_example_from_github from dagster._generate.generate import ( - generate_code_location as generate_code_location, generate_project as generate_project, generate_repository as generate_repository, ) diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index d51fb3f81a660..0266b2b59b3f4 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -11,11 +11,13 @@ # Examples aren't that can't be downloaded from the dagster project CLI EXAMPLES_TO_IGNORE = [ "deploy_k8s_beta", - "docs_snippets", "docs_beta_snippets", + "docs_snippets", "experimental", - "temp_pins.txt", "use_case_repository", + "pyproject.toml", + "README.md", + "temp_pins.txt", ] # Hardcoded list of available examples. The list is tested against the examples folder in this mono # repo to make sure it's up-to-date. diff --git a/python_modules/dagster/dagster/_generate/generate.py b/python_modules/dagster/dagster/_generate/generate.py index 07e336fc58a86..a3ce06873ea1b 100644 --- a/python_modules/dagster/dagster/_generate/generate.py +++ b/python_modules/dagster/dagster/_generate/generate.py @@ -1,12 +1,22 @@ import os import posixpath +from typing import List, Optional import click import jinja2 from dagster.version import __version__ as dagster_version -IGNORE_PATTERN_LIST = ["__pycache__", ".pytest_cache", "*.egg-info", ".DS_Store", "tox.ini"] +DEFAULT_EXCLUDES: List[str] = [ + "__pycache__", + ".pytest_cache", + "*.egg-info", + ".DS_Store", + ".ruff_cache", + "tox.ini", +] + +PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" def generate_repository(path: str): @@ -14,124 +24,100 @@ def generate_repository(path: str): click.echo(f"Creating a Dagster repository at {path}.") - # Step 1: Generate files for Dagster repository - _generate_files_from_template( + # + generate_project( path=path, + excludes=None, name_placeholder=REPO_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", REPO_NAME_PLACEHOLDER - ), + templates_path=os.path.join(os.path.dirname(__file__), "templates", REPO_NAME_PLACEHOLDER), ) click.echo(f"Generated files for Dagster repository in {path}.") -def generate_code_location(path: str): - CODE_LOCATION_NAME_PLACEHOLDER = "CODE_LOCATION_NAME_PLACEHOLDER" - - click.echo(f"Creating a Dagster code location at {path}.") - - # Step 1: Generate files for Dagster code location including pyproject.toml, setup.py - _generate_files_from_template( - path=path, - name_placeholder=CODE_LOCATION_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", CODE_LOCATION_NAME_PLACEHOLDER - ), - ) - - click.echo(f"Generated files for Dagster code location in {path}.") - - -def generate_project(path: str): - PROJECT_NAME_PLACEHOLDER = "PROJECT_NAME_PLACEHOLDER" +def generate_project( + path: str, + excludes: Optional[List[str]] = None, + name_placeholder: str = PROJECT_NAME_PLACEHOLDER, + templates_path: str = PROJECT_NAME_PLACEHOLDER, +): + """Renders templates for Dagster project.""" + excludes = DEFAULT_EXCLUDES if not excludes else DEFAULT_EXCLUDES + excludes click.echo(f"Creating a Dagster project at {path}.") - # Step 1: Generate files for Dagster code location - generate_code_location(path) - - # Step 2: Generate project-level files, e.g. README - _generate_files_from_template( - path=path, - name_placeholder=PROJECT_NAME_PLACEHOLDER, - project_template_path=os.path.join( - os.path.dirname(__file__), "templates", PROJECT_NAME_PLACEHOLDER - ), - skip_mkdir=True, - ) - - click.echo(f"Generated files for Dagster project in {path}.") - - -def _generate_files_from_template( - path: str, name_placeholder: str, project_template_path: str, skip_mkdir: bool = False -): normalized_path = os.path.normpath(path) - code_location_name = os.path.basename(normalized_path).replace("-", "_") + project_name: str = os.path.basename(normalized_path).replace("-", "_") + os.mkdir(normalized_path) - if not skip_mkdir: # skip if the dir is created by previous command - os.mkdir(normalized_path) - - loader = jinja2.FileSystemLoader(searchpath=project_template_path) - env = jinja2.Environment(loader=loader) + project_template_path: str = os.path.join( + os.path.dirname(__file__), "templates", templates_path + ) + loader: jinja2.loaders.FileSystemLoader = jinja2.FileSystemLoader( + searchpath=project_template_path + ) + env: jinja2.environment.Environment = jinja2.Environment(loader=loader) + # merge custom skip_files with the default list for root, dirs, files in os.walk(project_template_path): # For each subdirectory in the source template, create a subdirectory in the destination. for dirname in dirs: - src_dir_path = os.path.join(root, dirname) - if _should_skip_file(src_dir_path): + src_dir_path: str = os.path.join(root, dirname) + if _should_skip_file(src_dir_path, excludes): continue - src_relative_dir_path = os.path.relpath(src_dir_path, project_template_path) - dst_relative_dir_path = src_relative_dir_path.replace( + src_relative_dir_path: str = os.path.relpath(src_dir_path, project_template_path) + dst_relative_dir_path: str = src_relative_dir_path.replace( name_placeholder, - code_location_name, + project_name, 1, ) - dst_dir_path = os.path.join(normalized_path, dst_relative_dir_path) + dst_dir_path: str = os.path.join(normalized_path, dst_relative_dir_path) os.mkdir(dst_dir_path) # For each file in the source template, render a file in the destination. for filename in files: src_file_path = os.path.join(root, filename) - if _should_skip_file(src_file_path): + if _should_skip_file(src_file_path, excludes): continue - src_relative_file_path = os.path.relpath(src_file_path, project_template_path) - dst_relative_file_path = src_relative_file_path.replace( + src_relative_file_path: str = os.path.relpath(src_file_path, project_template_path) + dst_relative_file_path: str = src_relative_file_path.replace( name_placeholder, - code_location_name, + project_name, 1, ) - dst_file_path = os.path.join(normalized_path, dst_relative_file_path) + dst_file_path: str = os.path.join(normalized_path, dst_relative_file_path) - if dst_file_path.endswith(".tmpl"): - dst_file_path = dst_file_path[: -len(".tmpl")] + if dst_file_path.endswith(".jinja"): + dst_file_path = dst_file_path[: -len(".jinja")] with open(dst_file_path, "w", encoding="utf8") as f: # Jinja template names must use the POSIX path separator "/". - template_name = src_relative_file_path.replace(os.sep, posixpath.sep) - template = env.get_template(name=template_name) + template_name: str = src_relative_file_path.replace(os.sep, posixpath.sep) + template: jinja2.environment.Template = env.get_template(name=template_name) f.write( template.render( - repo_name=code_location_name, # deprecated - code_location_name=code_location_name, + repo_name=project_name, # deprecated + code_location_name=project_name, dagster_version=dagster_version, + project_name=project_name, ) ) f.write("\n") + click.echo(f"Generated files for Dagster project in {path}.") + -def _should_skip_file(path): +def _should_skip_file(path: str, excludes: List[str] = DEFAULT_EXCLUDES): """Given a file path `path` in a source template, returns whether or not the file should be skipped when generating destination files. Technically, `path` could also be a directory path that should be skipped. """ - for pattern in IGNORE_PATTERN_LIST: - if pattern in path: + for pattern in excludes: + if pattern.lower() in path.lower(): return True return False diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl deleted file mode 100644 index adfae7c0745fd..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = {{ code_location_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/assets.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/assets.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja similarity index 74% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja index 6dd330fcd8ded..1f59c3f583c8c 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja @@ -1,6 +1,6 @@ from dagster import Definitions, load_assets_from_modules -from . import assets # noqa: TID252 +from {{ project_name }} import assets # noqa: TID252 all_assets = load_assets_from_modules([assets]) diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/__init__.py rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/__init__.py diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja similarity index 100% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER_tests/test_assets.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/PROJECT_NAME_PLACEHOLDER_tests/test_assets.py.jinja diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja similarity index 83% rename from python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja index af2c1cc75743e..f1e6dde91d3bd 100644 --- a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/README.md.jinja @@ -1,4 +1,4 @@ -# {{ repo_name }} +# {{ project_name }} This is a [Dagster](https://dagster.io/) project scaffolded with [`dagster project scaffold`](https://docs.dagster.io/getting-started/create-new-project). @@ -18,7 +18,7 @@ dagster dev Open http://localhost:3000 with your browser to see the project. -You can start writing assets in `{{ repo_name }}/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. +You can start writing assets in `{{ project_name }}/assets.py`. The assets are automatically loaded into the Dagster code location as you define them. ## Development @@ -28,10 +28,10 @@ You can specify new Python dependencies in `setup.py`. ### Unit testing -Tests are in the `{{ repo_name }}_tests` directory and you can run tests using `pytest`: +Tests are in the `{{ project_name }}_tests` directory and you can run tests using `pytest`: ```bash -pytest {{ repo_name }}_tests +pytest {{ project_name }}_tests ``` ### Schedules and sensors diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja similarity index 66% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja index 84e7d57e93b61..6e1afe2858a3a 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja @@ -1,5 +1,5 @@ [project] -name = "{{ code_location_name }}" +name = "{{ project_name }}" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -20,5 +20,8 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] -module_name = "{{ code_location_name }}.definitions" -code_location_name = "{{ code_location_name }}" +module_name = "{{ project_name }}.definitions" +project_name = "{{ project_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ project_name }}_tests"] \ No newline at end of file diff --git a/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja new file mode 100644 index 0000000000000..dbfb6531ca2fb --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.cfg.jinja @@ -0,0 +1,2 @@ +[metadata] +name = {{ project_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja similarity index 63% rename from python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl rename to python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja index 043242d6313bb..114e9232e5749 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/PROJECT_NAME_PLACEHOLDER/setup.py.jinja @@ -1,8 +1,8 @@ from setuptools import find_packages, setup setup( - name="{{ code_location_name }}", - packages=find_packages(exclude=["{{ code_location_name }}_tests"]), + name="{{ project_name }}", + packages=find_packages(exclude=["{{ project_name }}_tests"]), install_requires=[ "dagster", "dagster-cloud" diff --git a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py index 7a0fbf9055520..f7318e1ad2533 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/test_project_commands.py @@ -1,5 +1,6 @@ import os import re +from pathlib import Path from click.testing import CliRunner from dagster import file_relative_path @@ -19,7 +20,8 @@ def test_project_scaffold_command_fails_when_dir_path_exists(): with runner.isolated_filesystem(): os.mkdir("existing_dir") result = runner.invoke(scaffold_command, ["--name", "existing_dir"]) - assert re.match(r"The directory .* already exists", result.output) + assert "The directory" in result.output + assert "already exists" in result.output assert result.exit_code != 0 @@ -39,11 +41,11 @@ def test_project_scaffold_command_succeeds(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_command, ["--name", "my_dagster_project"]) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/my_dagster_project") - assert os.path.exists("my_dagster_project/my_dagster_project_tests") - assert os.path.exists("my_dagster_project/README.md") - assert os.path.exists("my_dagster_project/pyproject.toml") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/my_dagster_project/__init__.py").exists() + assert Path("my_dagster_project/my_dagster_project_tests/__init__.py").exists() + assert Path("my_dagster_project/README.md").exists() + assert Path("my_dagster_project/pyproject.toml").exists() # test target loadable origins = get_origins_from_toml("my_dagster_project/pyproject.toml") @@ -51,29 +53,29 @@ def test_project_scaffold_command_succeeds(): assert origins[0].loadable_target_origin.module_name == "my_dagster_project.definitions" -def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): +def test_project_scaffold_command_excludes_succeeds(): runner = CliRunner() with runner.isolated_filesystem(): - os.mkdir("existing_dir") - result = runner.invoke(scaffold_code_location_command, ["--name", "existing_dir"]) - assert re.match(r"The directory .* already exists", result.output) - assert result.exit_code != 0 + result = runner.invoke( + scaffold_command, + ["--name", "diet_dagster", "--excludes", "setup", "--excludes", "tests"], + ) + assert result.exit_code == 0 + assert Path("diet_dagster/pyproject.toml").exists() + assert Path("diet_dagster/README.md").exists() + assert not Path("diet_dagster/diet_dagster_tests/__init__.py").exists() + assert not Path("diet_dagster/setup.cfg").exists() + assert not Path("diet_dagster/setup.py").exists() -def test_scaffold_code_location_command_succeeds(): +def test_project_scaffold_command_excludes_fails_on_required_files(): runner = CliRunner() with runner.isolated_filesystem(): - result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_code"]) - assert result.exit_code == 0 - assert os.path.exists("my_dagster_code") - assert os.path.exists("my_dagster_code/my_dagster_code") - assert os.path.exists("my_dagster_code/my_dagster_code_tests") - assert os.path.exists("my_dagster_code/pyproject.toml") - - # test target loadable - origins = get_origins_from_toml("my_dagster_code/pyproject.toml") - assert len(origins) == 1 - assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" + result = runner.invoke( + scaffold_command, + ["--name", "diet_dagster", "--excludes", "pyproject"], + ) + assert result.exit_code != 0 def test_from_example_command_fails_when_example_not_available(): @@ -94,11 +96,11 @@ def test_from_example_command_succeeds(): ["--name", "my_dagster_project", "--example", "assets_dbt_python"], ) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/assets_dbt_python") - assert os.path.exists("my_dagster_project/assets_dbt_python_tests") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/assets_dbt_python").exists() + assert Path("my_dagster_project/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("my_dagster_project/tox.ini") + assert not Path("my_dagster_project/tox.ini").exists() def test_from_example_command_versioned_succeeds(): @@ -116,11 +118,11 @@ def test_from_example_command_versioned_succeeds(): ], ) assert result.exit_code == 0 - assert os.path.exists("my_dagster_project") - assert os.path.exists("my_dagster_project/assets_dbt_python") - assert os.path.exists("my_dagster_project/assets_dbt_python_tests") + assert Path("my_dagster_project").exists() + assert Path("my_dagster_project/assets_dbt_python").exists() + assert Path("my_dagster_project/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("my_dagster_project/tox.ini") + assert not Path("my_dagster_project/tox.ini").exists() def test_from_example_command_default_name(): @@ -131,21 +133,21 @@ def test_from_example_command_default_name(): ["--name", "assets_dbt_python", "--example", "assets_dbt_python"], ) assert result.exit_code == 0 - assert os.path.exists("assets_dbt_python") - assert os.path.exists("assets_dbt_python/assets_dbt_python") - assert os.path.exists("assets_dbt_python/assets_dbt_python_tests") + assert Path("assets_dbt_python").exists() + assert Path("assets_dbt_python/assets_dbt_python").exists() + assert Path("assets_dbt_python/assets_dbt_python_tests").exists() # ensure we filter out tox.ini because it's used in our own CI - assert not os.path.exists("assets_dbt_python/tox.ini") + assert not Path("assets_dbt_python/tox.ini").exists() def test_available_examples_in_sync_with_example_folder(): # ensure the list of AVAILABLE_EXAMPLES is in sync with the example folder minus EXAMPLES_TO_IGNORE - example_folder = file_relative_path(__file__, "../../../../examples") + example_folder = Path(file_relative_path(__file__, "../../../../examples")) available_examples_in_folder = [ e for e in os.listdir(example_folder) if ( - os.path.isdir(os.path.join(example_folder, e)) + Path(example_folder / e).exists() and e not in EXAMPLES_TO_IGNORE and not _should_skip_file(e) ) @@ -153,6 +155,46 @@ def test_available_examples_in_sync_with_example_folder(): assert set(available_examples_in_folder) == set(AVAILABLE_EXAMPLES) +##################### +# `dagster project scaffold-code-location` command is deprecated. + + +def test_scaffold_code_location_deprecation(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_project"]) + assert re.match( + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", + result.output.lstrip(), + ) + + +def test_scaffold_code_location_scaffold_command_fails_when_dir_path_exists(): + runner = CliRunner() + with runner.isolated_filesystem(): + os.mkdir("existing_dir") + result = runner.invoke(scaffold_code_location_command, ["--name", "existing_dir"]) + assert "The directory" in result.output + assert "already exists" in result.output + assert result.exit_code != 0 + + +def test_scaffold_code_location_command_succeeds(): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke(scaffold_code_location_command, ["--name", "my_dagster_code"]) + assert result.exit_code == 0 + assert Path("my_dagster_code").exists() + assert Path("my_dagster_code/my_dagster_code/__init__.py").exists() + assert Path("my_dagster_code/my_dagster_code_tests/__init__.py").exists() + assert Path("my_dagster_code/pyproject.toml").exists() + + # test target loadable + origins = get_origins_from_toml("my_dagster_code/pyproject.toml") + assert len(origins) == 1 + assert origins[0].loadable_target_origin.module_name == "my_dagster_code.definitions" + + # `dagster project scaffold-repository` command is deprecated. # We're keeping the tests below for backcompat. @@ -162,8 +204,7 @@ def test_scaffold_repository_deprecation(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_project"]) assert re.match( - "WARNING: This command is deprecated. Use `dagster project scaffold-code-location`" - " instead.", + "WARNING: command is deprecated. Use `dagster project scaffold --excludes readme` instead.", result.output, ) @@ -182,10 +223,10 @@ def test_scaffold_repository_command_succeeds(): with runner.isolated_filesystem(): result = runner.invoke(scaffold_repository_command, ["--name", "my_dagster_repo"]) assert result.exit_code == 0 - assert os.path.exists("my_dagster_repo") - assert os.path.exists("my_dagster_repo/my_dagster_repo") - assert os.path.exists("my_dagster_repo/my_dagster_repo_tests") - assert not os.path.exists("my_dagster_repo/workspace.yaml") + assert Path("my_dagster_repo").exists() + assert Path("my_dagster_repo/my_dagster_repo").exists() + assert Path("my_dagster_repo/my_dagster_repo_tests").exists() + assert not Path("my_dagster_repo/workspace.yaml").exists() def test_versioned_download(): diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index d97c6bff7c7c9..494534db2687c 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -49,16 +49,22 @@ def main( # Supported on all Python versions. editable_target_paths = [ + ".buildkite/dagster-buildkite", + "examples/experimental/dagster-blueprints", + "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", + "integration_tests/python_modules/dagster-k8s-test-infra", + "helm/dagster/schema[test]", + "python_modules/automation", "python_modules/dagster[pyright,ruff,test]", "python_modules/dagster-pipes", "python_modules/dagster-graphql", "python_modules/dagster-test", "python_modules/dagster-webserver", "python_modules/dagit", - "python_modules/automation", "python_modules/libraries/dagster-managed-elements", "python_modules/libraries/dagster-airbyte", "python_modules/libraries/dagster-aws[stubs,test]", + "python_modules/libraries/dagster-azure", "python_modules/libraries/dagster-celery", "python_modules/libraries/dagster-celery-docker", "python_modules/libraries/dagster-dask[yarn,pbs,kube]", @@ -66,6 +72,9 @@ def main( "python_modules/libraries/dagster-datadog", "python_modules/libraries/dagster-datahub", "python_modules/libraries/dagster-dbt", + "python_modules/libraries/dagster-deltalake", + "python_modules/libraries/dagster-deltalake-pandas", + "python_modules/libraries/dagster-deltalake-polars", "python_modules/libraries/dagster-docker", "python_modules/libraries/dagster-gcp", "python_modules/libraries/dagster-gcp-pandas", @@ -77,31 +86,26 @@ def main( "python_modules/libraries/dagster-celery-k8s", "python_modules/libraries/dagster-github", "python_modules/libraries/dagster-mlflow", + "python_modules/libraries/dagster-msteams", "python_modules/libraries/dagster-mysql", "python_modules/libraries/dagster-looker", "python_modules/libraries/dagster-openai", "python_modules/libraries/dagster-pagerduty", "python_modules/libraries/dagster-pandas", + "python_modules/libraries/dagster-pandera", + "python_modules/libraries/dagster-polars[deltalake,gcp,test]", "python_modules/libraries/dagster-papertrail", "python_modules/libraries/dagster-postgres", "python_modules/libraries/dagster-prometheus", "python_modules/libraries/dagster-pyspark", "python_modules/libraries/dagster-shell", "python_modules/libraries/dagster-slack", + "python_modules/libraries/dagster-snowflake", + "python_modules/libraries/dagster-snowflake-pandas", "python_modules/libraries/dagster-spark", "python_modules/libraries/dagster-ssh", "python_modules/libraries/dagster-twilio", "python_modules/libraries/dagstermill", - "integration_tests/python_modules/dagster-k8s-test-infra", - "python_modules/libraries/dagster-azure", - "python_modules/libraries/dagster-msteams", - "python_modules/libraries/dagster-deltalake", - "python_modules/libraries/dagster-deltalake-pandas", - "python_modules/libraries/dagster-deltalake-polars", - "helm/dagster/schema[test]", - ".buildkite/dagster-buildkite", - "examples/experimental/dagster-blueprints", - "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", ] if sys.version_info <= (3, 12): @@ -114,13 +118,8 @@ def main( "python_modules/libraries/dagster-airflow", ] - if sys.version_info > (3, 7): - editable_target_paths += [ - "python_modules/libraries/dagster-pandera", - "python_modules/libraries/dagster-snowflake", - "python_modules/libraries/dagster-snowflake-pandas", - "python_modules/libraries/dagster-polars[deltalake,gcp,test]", - ] + # if sys.version_info > (3, 7): + # editable_target_paths += [] install_targets += list( itertools.chain.from_iterable(