From c5821443bac4fc05f65cc1e950058af32d072cfe Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Wed, 16 Oct 2024 08:42:33 +1100 Subject: [PATCH 1/3] refactor: consolidate config to pyproject.toml only --- .../getting-started/create-new-project.mdx | 2 +- .../understanding-dagster-project-files.mdx | 2 -- docs/content/integrations/dbt/quickstart.mdx | 2 -- .../docs/guides/project-structure.md | 3 +-- .../pyproject.toml.tmpl | 3 +++ .../setup.cfg.tmpl | 2 -- .../setup.py.tmpl | 11 -------- .../REPO_NAME_PLACEHOLDER/pyproject.toml | 3 --- .../REPO_NAME_PLACEHOLDER/pyproject.toml.tmpl | 27 +++++++++++++++++++ .../REPO_NAME_PLACEHOLDER/setup.cfg.tmpl | 2 -- .../REPO_NAME_PLACEHOLDER/setup.py.tmpl | 11 -------- 11 files changed, 32 insertions(+), 36 deletions(-) delete mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl delete mode 100644 python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl delete mode 100644 python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml create mode 100644 python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml.tmpl delete mode 100644 python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.cfg.tmpl delete mode 100644 python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.py.tmpl diff --git a/docs/content/getting-started/create-new-project.mdx b/docs/content/getting-started/create-new-project.mdx index 0d632583a881f..6e7bfeeda90d9 100644 --- a/docs/content/getting-started/create-new-project.mdx +++ b/docs/content/getting-started/create-new-project.mdx @@ -33,7 +33,7 @@ pip install dagster dagster project scaffold --name my-dagster-project ``` -The `dagster project scaffold` command generates a folder structure with a single Dagster code location and other files, such as `pyproject.toml` and `setup.py`. This takes care of setting things up with an empty project, enabling you to quickly get started. +The `dagster project scaffold` command generates a folder structure with a single Dagster code location and the canonical Python configuration file `pyproject.toml`. This empty project enables you to quickly get started. diff --git a/docs/content/guides/understanding-dagster-project-files.mdx b/docs/content/guides/understanding-dagster-project-files.mdx index c070217245b96..22e0718cb6f49 100644 --- a/docs/content/guides/understanding-dagster-project-files.mdx +++ b/docs/content/guides/understanding-dagster-project-files.mdx @@ -27,8 +27,6 @@ The following demonstrates a Dagster project using the default project skeleton, │   └── definitions.py ├── my_dagster_project_tests ├── pyproject.toml -├── setup.cfg -├── setup.py └── tox.ini ``` diff --git a/docs/content/integrations/dbt/quickstart.mdx b/docs/content/integrations/dbt/quickstart.mdx index c213eb3da3818..f803d4cda4f26 100644 --- a/docs/content/integrations/dbt/quickstart.mdx +++ b/docs/content/integrations/dbt/quickstart.mdx @@ -143,8 +143,6 @@ my_dagster_project ├── assets.py ├── definitions.py ├── pyproject.toml -├── setup.cfg -└── setup.py ``` 1. Change directories to the Dagster project directory: diff --git a/docs/docs-beta/docs/guides/project-structure.md b/docs/docs-beta/docs/guides/project-structure.md index bae009c702cd7..123c86fae5d03 100644 --- a/docs/docs-beta/docs/guides/project-structure.md +++ b/docs/docs-beta/docs/guides/project-structure.md @@ -29,8 +29,7 @@ $ dagster project scaffold --name example-dagster-project │   ├── __init__.py │   └── test_assets.py ├── pyproject.toml -├── setup.cfg -└── setup.py + ``` This is a great structure as you are first getting started, however, as you begin to introduce more assets, jobs, resources, sensors, and utility code, you may find that your Python files are growing too large to manage. diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl index 04c2b8c22e006..c4069805c38f0 100644 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl +++ b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -22,3 +22,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "{{ code_location_name }}.definitions" code_location_name = "{{ code_location_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ code_location_name }}_tests"] diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl deleted file mode 100644 index adfae7c0745fd..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.cfg.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = {{ code_location_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl b/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl deleted file mode 100644 index 043242d6313bb..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/CODE_LOCATION_NAME_PLACEHOLDER/setup.py.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="{{ code_location_name }}", - packages=find_packages(exclude=["{{ code_location_name }}_tests"]), - install_requires=[ - "dagster", - "dagster-cloud" - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml b/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml deleted file mode 100644 index 7fd26b970b848..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml.tmpl b/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml.tmpl new file mode 100644 index 0000000000000..2b03131540471 --- /dev/null +++ b/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/pyproject.toml.tmpl @@ -0,0 +1,27 @@ +[project] +name = "{{ repo_name }}" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-cloud", +] + +[project.optional-dependencies] +dev = [ + "dagster-webserver", + "pytest", +] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.dagster] +module_name = "{{ repo_name }}.definitions" +code_location_name = "{{ repo_name }}" + +[tool.setuptools.packages.find] +exclude=["{{ repo_name }}_tests"] diff --git a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.cfg.tmpl b/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.cfg.tmpl deleted file mode 100644 index a49ecc3865b3f..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.cfg.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = {{ repo_name }} diff --git a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.py.tmpl b/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.py.tmpl deleted file mode 100644 index 2ffd5b54ab8c6..0000000000000 --- a/python_modules/dagster/dagster/_generate/templates/REPO_NAME_PLACEHOLDER/setup.py.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="{{ repo_name }}", - packages=find_packages(exclude=["{{ repo_name }}_tests"]), - install_requires=[ - "dagster", - "dagster-cloud" - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) From b27489c2bb097dad09e1bc12860eb733d78e5d63 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sat, 19 Oct 2024 00:32:48 +1100 Subject: [PATCH 2/3] docs: consolidate setup config to pyproject.toml --- examples/assets_dbt_python/pyproject.toml | 30 ++++++++ examples/assets_dbt_python/setup.cfg | 2 - examples/assets_dbt_python/setup.py | 24 ------ .../assets_dynamic_partitions/pyproject.toml | 19 +++++ examples/assets_dynamic_partitions/setup.cfg | 2 - examples/assets_dynamic_partitions/setup.py | 12 --- .../assets_modern_data_stack/pyproject.toml | 36 +++++++++ examples/assets_modern_data_stack/setup.cfg | 2 - examples/assets_modern_data_stack/setup.py | 33 -------- examples/assets_pandas_pyspark/pyproject.toml | 20 +++++ examples/assets_pandas_pyspark/setup.cfg | 2 - examples/assets_pandas_pyspark/setup.py | 16 ---- .../pyproject.toml | 25 +++++- .../assets_pandas_type_metadata/setup.cfg | 2 - examples/assets_pandas_type_metadata/setup.py | 17 ----- examples/assets_smoke_test/pyproject.toml | 31 ++++++++ examples/assets_smoke_test/setup.cfg | 2 - examples/assets_smoke_test/setup.py | 19 ----- .../development_to_production/pyproject.toml | 30 ++++++++ examples/development_to_production/setup.cfg | 2 - examples/development_to_production/setup.py | 26 ------- examples/docs_beta_snippets/pyproject.toml | 27 +++++++ examples/docs_beta_snippets/setup.py | 19 ----- examples/docs_snippets/pyproject.toml | 76 +++++++++++++++++++ examples/docs_snippets/setup.py | 69 ----------------- .../assets_yaml_dsl/pyproject.toml | 30 +++++++- .../experimental/assets_yaml_dsl/setup.cfg | 2 - .../experimental/assets_yaml_dsl/setup.py | 17 ----- .../external_assets/pyproject.toml | 21 +++++ .../experimental/external_assets/setup.cfg | 2 - .../experimental/external_assets/setup.py | 8 -- .../pyproject.toml | 29 ++++++- examples/project_analytics/pyproject.toml | 27 +++++++ examples/project_analytics/setup.cfg | 2 - examples/project_analytics/setup.py | 21 ----- .../pyproject.toml | 23 ++++++ .../setup.cfg | 2 - .../project_dagster_university_start/setup.py | 17 ----- .../project_du_dbt_starter/pyproject.toml | 31 ++++++++ examples/project_du_dbt_starter/setup.cfg | 2 - examples/project_du_dbt_starter/setup.py | 24 ------ .../project_fully_featured/pyproject.toml | 55 +++++++++++++- examples/project_fully_featured/setup.cfg | 2 - examples/project_fully_featured/setup.py | 42 ---------- examples/quickstart_aws/pyproject.toml | 21 +++++ examples/quickstart_aws/setup.cfg | 2 - examples/quickstart_aws/setup.py | 15 ---- examples/quickstart_etl/pyproject.toml | 22 +++++- examples/quickstart_etl/setup.cfg | 2 - examples/quickstart_etl/setup.py | 14 ---- examples/quickstart_gcp/pyproject.toml | 27 +++++++ examples/quickstart_gcp/setup.cfg | 2 - examples/quickstart_gcp/setup.py | 21 ----- examples/quickstart_snowflake/pyproject.toml | 29 ++++++- examples/quickstart_snowflake/setup.cfg | 2 - examples/quickstart_snowflake/setup.py | 21 ----- examples/tutorial/pyproject.toml | 23 ++++++ examples/tutorial/setup.cfg | 2 - examples/tutorial/setup.py | 17 ----- .../tutorial_notebook_assets/pyproject.toml | 21 +++++ examples/tutorial_notebook_assets/setup.cfg | 2 - examples/tutorial_notebook_assets/setup.py | 15 ---- examples/use_case_repository/pyproject.toml | 28 ++++++- examples/use_case_repository/setup.cfg | 2 - examples/use_case_repository/setup.py | 26 ------- examples/with_airflow/pyproject.toml | 22 ++++++ examples/with_airflow/setup.cfg | 2 - examples/with_airflow/setup.py | 18 ----- .../with_great_expectations/pyproject.toml | 20 ++++- examples/with_great_expectations/setup.cfg | 2 - examples/with_great_expectations/setup.py | 12 --- examples/with_openai/pyproject.toml | 23 ++++++ examples/with_openai/setup.cfg | 2 - examples/with_openai/setup.py | 17 ----- examples/with_pyspark/pyproject.toml | 18 +++++ examples/with_pyspark/setup.cfg | 2 - examples/with_pyspark/setup.py | 12 --- examples/with_pyspark_emr/pyproject.toml | 20 ++++- examples/with_pyspark_emr/setup.cfg | 2 - examples/with_pyspark_emr/setup.py | 12 --- examples/with_wandb/pyproject.toml | 25 +++++- examples/with_wandb/setup.cfg | 2 - examples/with_wandb/setup.py | 17 ----- 83 files changed, 799 insertions(+), 643 deletions(-) delete mode 100644 examples/assets_dbt_python/setup.cfg delete mode 100644 examples/assets_dbt_python/setup.py delete mode 100644 examples/assets_dynamic_partitions/setup.cfg delete mode 100644 examples/assets_dynamic_partitions/setup.py delete mode 100644 examples/assets_modern_data_stack/setup.cfg delete mode 100644 examples/assets_modern_data_stack/setup.py delete mode 100644 examples/assets_pandas_pyspark/setup.cfg delete mode 100644 examples/assets_pandas_pyspark/setup.py delete mode 100644 examples/assets_pandas_type_metadata/setup.cfg delete mode 100644 examples/assets_pandas_type_metadata/setup.py delete mode 100644 examples/assets_smoke_test/setup.cfg delete mode 100644 examples/assets_smoke_test/setup.py delete mode 100644 examples/development_to_production/setup.cfg delete mode 100644 examples/development_to_production/setup.py delete mode 100755 examples/docs_beta_snippets/setup.py delete mode 100755 examples/docs_snippets/setup.py delete mode 100644 examples/experimental/assets_yaml_dsl/setup.cfg delete mode 100644 examples/experimental/assets_yaml_dsl/setup.py delete mode 100644 examples/experimental/external_assets/setup.cfg delete mode 100644 examples/experimental/external_assets/setup.py delete mode 100644 examples/project_analytics/setup.cfg delete mode 100644 examples/project_analytics/setup.py delete mode 100644 examples/project_dagster_university_start/setup.cfg delete mode 100644 examples/project_dagster_university_start/setup.py delete mode 100644 examples/project_du_dbt_starter/setup.cfg delete mode 100644 examples/project_du_dbt_starter/setup.py delete mode 100644 examples/project_fully_featured/setup.cfg delete mode 100644 examples/project_fully_featured/setup.py delete mode 100644 examples/quickstart_aws/setup.cfg delete mode 100644 examples/quickstart_aws/setup.py delete mode 100644 examples/quickstart_etl/setup.cfg delete mode 100644 examples/quickstart_etl/setup.py delete mode 100644 examples/quickstart_gcp/setup.cfg delete mode 100644 examples/quickstart_gcp/setup.py delete mode 100644 examples/quickstart_snowflake/setup.cfg delete mode 100644 examples/quickstart_snowflake/setup.py delete mode 100644 examples/tutorial/setup.cfg delete mode 100644 examples/tutorial/setup.py delete mode 100644 examples/tutorial_notebook_assets/setup.cfg delete mode 100644 examples/tutorial_notebook_assets/setup.py delete mode 100644 examples/use_case_repository/setup.cfg delete mode 100644 examples/use_case_repository/setup.py delete mode 100644 examples/with_airflow/setup.cfg delete mode 100644 examples/with_airflow/setup.py delete mode 100644 examples/with_great_expectations/setup.cfg delete mode 100644 examples/with_great_expectations/setup.py delete mode 100644 examples/with_openai/setup.cfg delete mode 100644 examples/with_openai/setup.py delete mode 100644 examples/with_pyspark/setup.cfg delete mode 100644 examples/with_pyspark/setup.py delete mode 100644 examples/with_pyspark_emr/setup.cfg delete mode 100644 examples/with_pyspark_emr/setup.py delete mode 100644 examples/with_wandb/setup.cfg delete mode 100644 examples/with_wandb/setup.py diff --git a/examples/assets_dbt_python/pyproject.toml b/examples/assets_dbt_python/pyproject.toml index f5fdf2bf897fd..aa233df26d49b 100644 --- a/examples/assets_dbt_python/pyproject.toml +++ b/examples/assets_dbt_python/pyproject.toml @@ -1,3 +1,27 @@ +[project] +name = "assets_dbt_python" +version = "0.1.0" +description = "Example Dagster project demonstrating dbt." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "boto3", + "dagster", + "dagster-cloud", + "dagster-dbt", + "dagster-duckdb", + "dagster-duckdb-pandas", + "dbt-core", + "dbt-duckdb", + "numpy", + "packaging<22.0", # packaging v22 has build compatibility issues with dbt as of 2022-12-07 + "pandas", + "scipy", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +29,9 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_dbt_python.definitions" code_location_name = "assets_dbt_python" + +[tool.setuptools.packages.find] +exclude = ["assets_dbt_python_tests"] + +[tool.setuptools.package-data] +assets_dbt_python = ["dbt-project/**/*"] diff --git a/examples/assets_dbt_python/setup.cfg b/examples/assets_dbt_python/setup.cfg deleted file mode 100644 index 354dae15c96e8..0000000000000 --- a/examples/assets_dbt_python/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_dbt_python diff --git a/examples/assets_dbt_python/setup.py b/examples/assets_dbt_python/setup.py deleted file mode 100644 index 8071544f606fe..0000000000000 --- a/examples/assets_dbt_python/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_dbt_python", - packages=find_packages(exclude=["assets_dbt_python_tests"]), - # package data paths are relative to the package key - package_data={"assets_dbt_python": ["dbt-project/**/*"]}, - install_requires=[ - "dagster", - "dagster-cloud", - "boto3", - "dagster-dbt", - "pandas", - "numpy", - "scipy", - "dbt-core", - "dbt-duckdb", - "dagster-duckdb", - "dagster-duckdb-pandas", - # packaging v22 has build compatibility issues with dbt as of 2022-12-07 - "packaging<22.0", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/assets_dynamic_partitions/pyproject.toml b/examples/assets_dynamic_partitions/pyproject.toml index e078eb17a0843..c13895165b928 100644 --- a/examples/assets_dynamic_partitions/pyproject.toml +++ b/examples/assets_dynamic_partitions/pyproject.toml @@ -1,3 +1,19 @@ +[project] +name = "assets_dynamic_partitions" +version = "0.1.0" +description = "Example Dagster project demonstrating dynamic partitions." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-cloud", + "dagster-duckdb-pandas", + "requests", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +21,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_dynamic_partitions.definitions" code_location_name = "assets_dynamic_partitions" + +[tool.setuptools.packages.find] +exclude = ["assets_dynamic_partitions_tests"] diff --git a/examples/assets_dynamic_partitions/setup.cfg b/examples/assets_dynamic_partitions/setup.cfg deleted file mode 100644 index 8be1939fd0742..0000000000000 --- a/examples/assets_dynamic_partitions/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_dynamic_partitions diff --git a/examples/assets_dynamic_partitions/setup.py b/examples/assets_dynamic_partitions/setup.py deleted file mode 100644 index c547f4e387282..0000000000000 --- a/examples/assets_dynamic_partitions/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_dynamic_partitions", - packages=find_packages(exclude=["assets_dynamic_partitions_tests"]), - install_requires=[ - "dagster", - "dagster-duckdb-pandas", - "requests", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/assets_modern_data_stack/pyproject.toml b/examples/assets_modern_data_stack/pyproject.toml index ef324b1d09174..9527498dc7eb5 100644 --- a/examples/assets_modern_data_stack/pyproject.toml +++ b/examples/assets_modern_data_stack/pyproject.toml @@ -1,3 +1,33 @@ +[project] +name = "assets_modern_data_stack" +version = "0.1.0" +description = "Example Dagster project demonstrating modern data stack." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "boto3", + "dagster", + "dagster-airbyte", + "dagster-cloud", + "dagster-dbt", + "dagster-postgres", + "dbt-core", + "dbt-postgres", + "numpy", + "pandas", + "scipy", + "packaging<22.0", # match dbt-core's requirement to workaround a resolution issue +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] +test = [ + # cant build psycopg2 in buildkite + # something about the 1.8.0 dependency setup to avoid psycopg2-binary on linux + # seems to prevent that dependency from being used even if explicitly added + "dbt-postgres<1.8.0" +] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +35,9 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_modern_data_stack.definitions" code_location_name = "assets_modern_data_stack" + +[tool.setuptools.packages.find] +exclude = ["assets_modern_data_stack_tests"] + +[tool.setuptools.package-data] +assets_modern_data_stack = ["../dbt_project/*", "../dbt_project/*/*"] diff --git a/examples/assets_modern_data_stack/setup.cfg b/examples/assets_modern_data_stack/setup.cfg deleted file mode 100644 index 2f1222e4749d4..0000000000000 --- a/examples/assets_modern_data_stack/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_modern_data_stack diff --git a/examples/assets_modern_data_stack/setup.py b/examples/assets_modern_data_stack/setup.py deleted file mode 100644 index 419a06850ae31..0000000000000 --- a/examples/assets_modern_data_stack/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_modern_data_stack", - packages=find_packages(exclude=["assets_modern_data_stack_tests"]), - package_data={"assets_modern_data_stack": ["../dbt_project/*", "../dbt_project/*/*"]}, - install_requires=[ - "dagster", - "dagster-cloud", - "boto3", - "dagster-airbyte", - "dagster-dbt", - "dagster-postgres", - "pandas", - "numpy", - "scipy", - "dbt-core", - "dbt-postgres", - "packaging<22.0", # match dbt-core's requirement to workaround a resolution issue - ], - extras_require={ - "dev": [ - "dagster-webserver", - "pytest", - ], - "test": [ - # cant build psycopg2 in buildkite - # something about the 1.8.0 dependency setup to avoid psycopg2-binary on linux - # seems to prevent that dependency from being used even if explicitly added - "dbt-postgres<1.8.0" - ], - }, -) diff --git a/examples/assets_pandas_pyspark/pyproject.toml b/examples/assets_pandas_pyspark/pyproject.toml index af5bb9b1aebc4..d54628801cee1 100644 --- a/examples/assets_pandas_pyspark/pyproject.toml +++ b/examples/assets_pandas_pyspark/pyproject.toml @@ -1,3 +1,20 @@ +[project] +name = "assets_pandas_pyspark" +version = "0.1.0" +description = "Example Dagster project demonstrating pandas and pyspark." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "pandas", + "pyspark", + # "pyarrow", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] +test = ["pandas", "pyarrow; python_version < '3.9'", "pyspark"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +22,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_pandas_pyspark.definitions" code_location_name = "assets_pandas_pyspark" + +[tool.setuptools.packages.find] +exclude = ["assets_pandas_pyspark_tests"] \ No newline at end of file diff --git a/examples/assets_pandas_pyspark/setup.cfg b/examples/assets_pandas_pyspark/setup.cfg deleted file mode 100644 index 11129b47ee2ef..0000000000000 --- a/examples/assets_pandas_pyspark/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_pandas_pyspark diff --git a/examples/assets_pandas_pyspark/setup.py b/examples/assets_pandas_pyspark/setup.py deleted file mode 100644 index 035e9bf9cba4c..0000000000000 --- a/examples/assets_pandas_pyspark/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_pandas_pyspark", - packages=find_packages(exclude=["assets_pandas_pyspark_tests"]), - install_requires=[ - "dagster", - "pandas", - "pyspark", - # "pyarrow", - ], - extras_require={ - "dev": ["dagster-webserver", "pytest"], - "test": ["pandas", "pyarrow; python_version < '3.9'", "pyspark"], - }, -) diff --git a/examples/assets_pandas_type_metadata/pyproject.toml b/examples/assets_pandas_type_metadata/pyproject.toml index e0843eb9285c4..881a34fd8f5a8 100644 --- a/examples/assets_pandas_type_metadata/pyproject.toml +++ b/examples/assets_pandas_type_metadata/pyproject.toml @@ -1,7 +1,30 @@ +[project] +name = "assets_pandas_type_metadata" +version = "0.1.0" +description = "Example Dagster project demonstrating pandas type metadata." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-pandera", + "jupyterlab", + "matplotlib", + "pandas", + "pandera", + "pyarrow", + "seaborn", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_pandas_type_metadata.definitions" -code_location_name = "assets_pandas_type_metadata" \ No newline at end of file +code_location_name = "assets_pandas_type_metadata" + +[tool.setuptools.packages.find] +exclude = ["assets_pandas_type_metadata_tests"] diff --git a/examples/assets_pandas_type_metadata/setup.cfg b/examples/assets_pandas_type_metadata/setup.cfg deleted file mode 100644 index 53f53630f43b7..0000000000000 --- a/examples/assets_pandas_type_metadata/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_pandas_type_metadata diff --git a/examples/assets_pandas_type_metadata/setup.py b/examples/assets_pandas_type_metadata/setup.py deleted file mode 100644 index e627fdb224203..0000000000000 --- a/examples/assets_pandas_type_metadata/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_pandas_type_metadata", - packages=find_packages(exclude=["assets_pandas_type_metadata_tests"]), - install_requires=[ - "dagster", - "dagster-pandera", - "jupyterlab", - "matplotlib", - "seaborn", - "pandera", - "pandas", - "pyarrow", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/assets_smoke_test/pyproject.toml b/examples/assets_smoke_test/pyproject.toml index 0905351f906c9..c83cd63d13f90 100644 --- a/examples/assets_smoke_test/pyproject.toml +++ b/examples/assets_smoke_test/pyproject.toml @@ -1,3 +1,28 @@ +[project] +name = "assets_smoke_test" +version = "0.1.0" +description = "Example Dagster project demonstrating assets smoke tests." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-dbt", + "dagster-pandas", + "dagster-snowflake", + "dagster-snowflake-pandas", + "dbt-snowflake", + "pandas", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] +test = [ + # cant build psycopg2 in buildkite + # something about the 1.8.0 dependency setup to avoid psycopg2-binary on linux + # seems to prevent that dependency from being used even if explicitly added + "dbt-postgres<1.8.0" +] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +30,9 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_smoke_test.definitions" code_location_name = "assets_smoke_test" + +[tool.setuptools.packages.find] +exclude = ["assets_smoke_test_tests"] + +[tool.setuptools.package-data] +assets_smoke_test = ["dbt_project/*"] diff --git a/examples/assets_smoke_test/setup.cfg b/examples/assets_smoke_test/setup.cfg deleted file mode 100644 index f3c52d44adf15..0000000000000 --- a/examples/assets_smoke_test/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_smoke_test diff --git a/examples/assets_smoke_test/setup.py b/examples/assets_smoke_test/setup.py deleted file mode 100644 index d898ee11fd2f1..0000000000000 --- a/examples/assets_smoke_test/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_smoke_test", - packages=find_packages(exclude=["assets_smoke_test_tests"]), - package_data={"assets_smoke_test": ["dbt_project/*"]}, - install_requires=[ - "dagster", - "dagster-pandas", - "dagster-dbt", - "pandas", - "dbt-snowflake", - "dagster-snowflake", - "dagster-snowflake-pandas", - ], - extras_require={ - "dev": ["dagster-webserver", "pytest"], - }, -) diff --git a/examples/development_to_production/pyproject.toml b/examples/development_to_production/pyproject.toml index bf1da349d188f..f4d1877d2b9ad 100644 --- a/examples/development_to_production/pyproject.toml +++ b/examples/development_to_production/pyproject.toml @@ -1,3 +1,33 @@ +[project] +name = "development_to_production" +version = "1!0+dev" +authors = [ + {name = "Dagster Labs", email = "hello@dagsterlabs.com"} +] +description = "Dagster example of local development and production deployment." +readme = "README.md" +requires-python = ">=3.8,<3.13" +license = "Apache-2.0" +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = [ + "dagster", + "dagster-snowflake", + "dagster-snowflake-pandas", + "pandas", + "requests", +] + +[project.urls] +url = "https://github.com/dagster-io/dagster/tree/master/examples/development_to_production" + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" diff --git a/examples/development_to_production/setup.cfg b/examples/development_to_production/setup.cfg deleted file mode 100644 index 324fe1b2a57b5..0000000000000 --- a/examples/development_to_production/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = development_to_production \ No newline at end of file diff --git a/examples/development_to_production/setup.py b/examples/development_to_production/setup.py deleted file mode 100644 index a3e21c2be2f1b..0000000000000 --- a/examples/development_to_production/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -from setuptools import setup - -setup( - name="development_to_production", - version="1!0+dev", - author_email="hello@dagsterlabs.com", - packages=["development_to_production"], # same as name - install_requires=[ - "dagster", - "dagster-snowflake", - "dagster-snowflake-pandas", - "pandas", - "requests", - ], # external packages as dependencies - author="Dagster Labs", - license="Apache-2.0", - description="Dagster example of local development and production deployment.", - url="https://github.com/dagster-io/dagster/tree/master/examples/development_to_production", - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/docs_beta_snippets/pyproject.toml b/examples/docs_beta_snippets/pyproject.toml index 1cac9a2823ffe..de4c51b908840 100644 --- a/examples/docs_beta_snippets/pyproject.toml +++ b/examples/docs_beta_snippets/pyproject.toml @@ -1,3 +1,27 @@ +[project] +name = "docs_beta_snippets" +version = "0.1.0" +authors = [ + { name = "Dagster Labs", email = "hello@dagsterlabs.com"} +] +license = "Apache-2.0" +description = "Dagster project for docs beta snippets." +readme = "README.md" +requires-python = ">=3.8,<3.13" +classifiers=[ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] + +[project.urls] +url = "https://github.com/dagster-io/dagster/tree/master/examples/docs_beta_snippets" + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [tool.ruff] # Extend example root configuration. @@ -33,3 +57,6 @@ extend-ignore = [ # Ensures ruff classifies imports from `dagster` as first-party. Keeps snippet imports relatively # compressed. known-first-party = ["dagster"] + +[tool.setuptools.packages.find] +exclude = ["docs_beta_snippets_tests"] diff --git a/examples/docs_beta_snippets/setup.py b/examples/docs_beta_snippets/setup.py deleted file mode 100755 index 9b790f6a51754..0000000000000 --- a/examples/docs_beta_snippets/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="docs_beta_snippets", - author="Dagster Labs", - author_email="hello@dagsterlabs.com", - license="Apache-2.0", - url="https://github.com/dagster-io/dagster/tree/master/examples/docs_beta_snippets", - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - packages=find_packages(exclude=["test"]), - # package not expected to be installed alone, dependencies for use managed in tox.ini - install_requires=[], -) diff --git a/examples/docs_snippets/pyproject.toml b/examples/docs_snippets/pyproject.toml index 1cac9a2823ffe..1d7a30579ea68 100644 --- a/examples/docs_snippets/pyproject.toml +++ b/examples/docs_snippets/pyproject.toml @@ -1,3 +1,76 @@ +[project] +name = "docs_snippets" +description = "Dagster project for docs snippets" +version = "0.1.0" +authors = [ + {name = "Dagster Labs", email = "hello@dagsterlabs.com"} +] +license = "Apache-2.0" +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-webserver", + "dagstermill", + "dagster-airbyte", + "dagster-airflow", + "dagster-aws", + "dagster-celery", + "dagster-dbt", + "dagster-dask", + "dagster-databricks", + "dagster-deltalake", + "dagster-deltalake-pandas", + "dagster-deltalake-polars", + "dagster-duckdb", + "dagster-duckdb-pandas", + "dagster-embedded-elt", + "dagster-fivetran", + "dagster-gcp", + "dagster-gcp-pandas", + "dagster-gcp-pyspark", + "dagster-graphql", + "dagster-k8s", + "dagster-postgres", + "dagster-slack", + "dagster-snowflake", + "dagster-snowflake-pandas", +] + +[project.urls] +url = "https://github.com/dagster-io/dagster/tree/master/examples/docs_snippets" + +[project.optional-dependencies] +full = [ + "click", + "dagster-webserver[test]", + "dbt-duckdb", + "matplotlib", + # matplotlib-inline 0.1.5 is causing mysterious + # "'NoneType' object has no attribute 'canvas'" errors in the tests that involve + # Jupyter notebooks + "matplotlib-inline<=0.1.3", + "moto<5.0", + "numpy", + "pandas", + "pandera", + "plotly", + "pytest", + "requests", + "seaborn", + "scikit-learn", + "slack_sdk", + "syrupy", + "xgboost", +] + [tool.ruff] # Extend example root configuration. @@ -33,3 +106,6 @@ extend-ignore = [ # Ensures ruff classifies imports from `dagster` as first-party. Keeps snippet imports relatively # compressed. known-first-party = ["dagster"] + +[tool.setuptools.packages.find] +exclude = ["docs_snippets_tests"] diff --git a/examples/docs_snippets/setup.py b/examples/docs_snippets/setup.py deleted file mode 100755 index fb71b29ec4f4d..0000000000000 --- a/examples/docs_snippets/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="docs_snippets", - author="Dagster Labs", - author_email="hello@dagsterlabs.com", - license="Apache-2.0", - url="https://github.com/dagster-io/dagster/tree/master/examples/docs_snippets", - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - packages=find_packages(exclude=["test"]), - install_requires=[ - "dagster", - "dagster-webserver", - "dagstermill", - "dagster-airbyte", - "dagster-airflow", - "dagster-aws", - "dagster-celery", - "dagster-dbt", - "dagster-dask", - "dagster-databricks", - "dagster-deltalake", - "dagster-deltalake-pandas", - "dagster-deltalake-polars", - "dagster-duckdb", - "dagster-duckdb-pandas", - "dagster-embedded-elt", - "dagster-fivetran", - "dagster-gcp", - "dagster-graphql", - "dagster-k8s", - "dagster-postgres", - "dagster-slack", - "dagster-gcp-pandas", - "dagster-gcp-pyspark", - "dagster-snowflake", - "dagster-snowflake-pandas", - ], - extras_require={ - "full": [ - "click", - "matplotlib", - # matplotlib-inline 0.1.5 is causing mysterious - # "'NoneType' object has no attribute 'canvas'" errors in the tests that involve - # Jupyter notebooks - "matplotlib-inline<=0.1.3", - "moto<5.0", - "numpy", - "pandas", - "pandera", - "plotly", - "pytest", - "requests", - "seaborn", - "scikit-learn", - "slack_sdk", - "syrupy", - "dbt-duckdb", - "xgboost", - "dagster-webserver[test]", - ] - }, -) diff --git a/examples/experimental/assets_yaml_dsl/pyproject.toml b/examples/experimental/assets_yaml_dsl/pyproject.toml index abdf8f2bb5a27..4dee8cdf6f06c 100644 --- a/examples/experimental/assets_yaml_dsl/pyproject.toml +++ b/examples/experimental/assets_yaml_dsl/pyproject.toml @@ -1,7 +1,35 @@ +[project] +name = "assets_yaml_dsl" +version = "0.1.0" +license = "Apache-2.0" +description = "Dagster example of yaml dsl for building asset graphs" +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-pipes", + "pandas", +] + +[project.urls] +url = "https://github.com/dagster-io/dagster/tree/master/examples/assets_yaml_dsl" + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "assets_yaml_dsl.definitions" -code_location_name = "assets_yaml_dsl" \ No newline at end of file +code_location_name = "assets_yaml_dsl" + +[tool.setuptools.packages.find] +exclude = ["assets_yaml_dsl_tests"] diff --git a/examples/experimental/assets_yaml_dsl/setup.cfg b/examples/experimental/assets_yaml_dsl/setup.cfg deleted file mode 100644 index 77b6eb1c4abab..0000000000000 --- a/examples/experimental/assets_yaml_dsl/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = assets_yaml_dsl diff --git a/examples/experimental/assets_yaml_dsl/setup.py b/examples/experimental/assets_yaml_dsl/setup.py deleted file mode 100644 index 1ec0f58a74a44..0000000000000 --- a/examples/experimental/assets_yaml_dsl/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="assets_yaml_dsl", - packages=find_packages(exclude=["assets_yaml_dsl_tests"]), - install_requires=["dagster", "pandas", "dagster-pipes"], - license="Apache-2.0", - description="Dagster example of yaml dsl for building asset graphs", - url="https://github.com/dagster-io/dagster/tree/master/examples/assets_yaml_dsl", - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/experimental/external_assets/pyproject.toml b/examples/experimental/external_assets/pyproject.toml index fed528d4a7a14..f328124003abc 100644 --- a/examples/experimental/external_assets/pyproject.toml +++ b/examples/experimental/external_assets/pyproject.toml @@ -1,3 +1,24 @@ +[project] +name = "external_assets" +version = "0.1.0" +license = "Apache-2.0" +description = "Dagster example of yaml dsl for building external_assets" +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster>=1.5.1", + "dagster-k8s>=0.21.1", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" diff --git a/examples/experimental/external_assets/setup.cfg b/examples/experimental/external_assets/setup.cfg deleted file mode 100644 index e6f98f026f73a..0000000000000 --- a/examples/experimental/external_assets/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = external_assets diff --git a/examples/experimental/external_assets/setup.py b/examples/experimental/external_assets/setup.py deleted file mode 100644 index 675b2c1c0ac3b..0000000000000 --- a/examples/experimental/external_assets/setup.py +++ /dev/null @@ -1,8 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="external_assets", - packages=find_packages(), - install_requires=["dagster>=1.5.1", "dagster-k8s>=0.21.1"], - extras_require={"dev": ["dagit", "pytest"]}, -) diff --git a/examples/feature_graph_backed_assets/pyproject.toml b/examples/feature_graph_backed_assets/pyproject.toml index dddd1b6daa982..f1c961be61c9f 100644 --- a/examples/feature_graph_backed_assets/pyproject.toml +++ b/examples/feature_graph_backed_assets/pyproject.toml @@ -1,7 +1,34 @@ +[project] +name = "feature_graph_backed_assets" +version = "0.1.0" +license = "Apache-2.0" +description = "Dagster example of op and graph-backed assets." +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "pandas", +] + +[project.urls] +url = "https://github.com/dagster-io/dagster/tree/master/examples/feature_graph_backed_assets" + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "feature_graph_backed_assets.definitions" -code_location_name = "feature_graph_backed_assets" \ No newline at end of file +code_location_name = "feature_graph_backed_assets" + +[tool.setuptools.packages.find] +exclude = ["feature_graph_backed_assets_tests"] diff --git a/examples/project_analytics/pyproject.toml b/examples/project_analytics/pyproject.toml index 85a91f019dd6b..f42c6be5ced8c 100644 --- a/examples/project_analytics/pyproject.toml +++ b/examples/project_analytics/pyproject.toml @@ -1,3 +1,27 @@ +[project] +name = "project_analytics" +version = "0.1.0" +description = "Dagster project for Pypi." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-cloud", + "dagster-dbt", + "dagster-duckdb-pandas", + "dagster-gcp", + "dagster-gcp-pandas", + "dagster-hex", + "dagster-pandas", + "dbt-bigquery", + "dbt-duckdb", + "google-cloud-bigquery", + "psycopg2-binary", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -23,3 +47,6 @@ extend-ignore = [ "TID252" ] + +[tool.setuptools.packages.find] +exclude = ["dagster_pypi_tests"] diff --git a/examples/project_analytics/setup.cfg b/examples/project_analytics/setup.cfg deleted file mode 100644 index fc2d5a830ca80..0000000000000 --- a/examples/project_analytics/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = dagster_pypi diff --git a/examples/project_analytics/setup.py b/examples/project_analytics/setup.py deleted file mode 100644 index 13290194c2c68..0000000000000 --- a/examples/project_analytics/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="dagster_pypi", - packages=find_packages(exclude=["dagster_pypi_tests"]), - install_requires=[ - "dagster", - "dagster-cloud", - "dagster-dbt", - "dagster-duckdb-pandas", - "dagster-gcp", - "dagster-gcp-pandas", - "dagster-hex", - "dagster-pandas", - "dbt-bigquery", - "dbt-duckdb", - "google-cloud-bigquery", - "psycopg2-binary", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/project_dagster_university_start/pyproject.toml b/examples/project_dagster_university_start/pyproject.toml index e7cb4a2244689..42e70147b9144 100644 --- a/examples/project_dagster_university_start/pyproject.toml +++ b/examples/project_dagster_university_start/pyproject.toml @@ -1,3 +1,23 @@ +[project] +name = "dagster_university" +version = "0.1.0" +description = "Dagster project for AWS quickstart." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster==1.7.*", + "dagster-cloud", + "dagster-duckdb", + "geopandas", + "kaleido", + "pandas", + "plotly", + "shapely", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -20,3 +40,6 @@ extend-ignore = [ "TID252" ] + +[tool.setuptools.packages.find] +exclude = ["dagster_university_tests"] diff --git a/examples/project_dagster_university_start/setup.cfg b/examples/project_dagster_university_start/setup.cfg deleted file mode 100644 index 95fae09b67690..0000000000000 --- a/examples/project_dagster_university_start/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = dagster_university diff --git a/examples/project_dagster_university_start/setup.py b/examples/project_dagster_university_start/setup.py deleted file mode 100644 index 83c1c99a3d0c4..0000000000000 --- a/examples/project_dagster_university_start/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="dagster_university", - packages=find_packages(exclude=["dagster_university_tests"]), - install_requires=[ - "dagster==1.7.*", - "dagster-cloud", - "dagster-duckdb", - "geopandas", - "kaleido", - "pandas", - "plotly", - "shapely", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/project_du_dbt_starter/pyproject.toml b/examples/project_du_dbt_starter/pyproject.toml index dc1a339508467..371bacf482d0b 100644 --- a/examples/project_du_dbt_starter/pyproject.toml +++ b/examples/project_du_dbt_starter/pyproject.toml @@ -1,6 +1,37 @@ +[project] +name = "dagster_university" +version = "0.1.0" +description = "Dagster university project" +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "boto3", + "dagster==1.7.*", + "dagster-cloud", + "dagster-duckdb", + "dagster-dbt", + "dbt-duckdb", + "geopandas", + "kaleido", + "pandas[parquet]", + "plotly", + "pyarrow", + "s3fs", + "shapely", + "smart_open[s3]", + "smart_open", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "dagster_university" + +[tool.setuptools.packages.find] +exclude = ["dagster_university_tests"] +where = ["dagster_university"] diff --git a/examples/project_du_dbt_starter/setup.cfg b/examples/project_du_dbt_starter/setup.cfg deleted file mode 100644 index 95fae09b67690..0000000000000 --- a/examples/project_du_dbt_starter/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = dagster_university diff --git a/examples/project_du_dbt_starter/setup.py b/examples/project_du_dbt_starter/setup.py deleted file mode 100644 index d2e3eee837bcf..0000000000000 --- a/examples/project_du_dbt_starter/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="dagster_university", - packages=find_packages(exclude=["dagster_university_tests"]), - install_requires=[ - "dagster==1.7.*", - "dagster-cloud", - "dagster-duckdb", - "dagster-dbt", - "dbt-duckdb", - "geopandas", - "kaleido", - "pandas[parquet]", - "plotly", - "shapely", - "smart_open[s3]", - "s3fs", - "smart_open", - "boto3", - "pyarrow", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/project_fully_featured/pyproject.toml b/examples/project_fully_featured/pyproject.toml index f31b224d258d0..66fc7c40cb5aa 100644 --- a/examples/project_fully_featured/pyproject.toml +++ b/examples/project_fully_featured/pyproject.toml @@ -1,7 +1,60 @@ +[project] +name = "project_fully_featured" +description = "Example fully featured Dagster project." +version = "1!0+dev" +authors = [ + {name = "Dagster Labs", email="hello@dagsterlabs.com"} +] +classifiers = [ + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: OS Independent", +] +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "aiobotocore", + "dagster", + "dagster-aws", + "dagster-dbt", + "dagster-pandas", + "dagster-postgres", + "dagster-pyspark", + "dagster-slack", + "dbt-duckdb", + "dbt-snowflake", + "duckdb!=0.3.3, <= 6.0.0", # missing wheels + "gcsfs", + "fsspec", + "mock", + "pandas", + "pyarrow>=4.0.0", + "pyspark", + "requests", + "s3fs", + "scipy", + "scikit-learn", + "snowflake-sqlalchemy<1.5.2", + "sqlalchemy!=1.4.42", # workaround for https://github.com/snowflakedb/snowflake-sqlalchemy/issues/350 +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] +tests = ["mypy", "pylint", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "project_fully_featured.definitions" -code_location_name = "project_fully_featured" \ No newline at end of file +code_location_name = "project_fully_featured" + +[tool.setuptools.package-data] +project_fully_featured = ["hacker_news_dbt/*"] + +[tool.setuptools.packages.find] +exclude = ["project_fully_featured_tests"] +where = ["project_fully_featured"] diff --git a/examples/project_fully_featured/setup.cfg b/examples/project_fully_featured/setup.cfg deleted file mode 100644 index 3e8e9b33b7d1c..0000000000000 --- a/examples/project_fully_featured/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = project_fully_featured diff --git a/examples/project_fully_featured/setup.py b/examples/project_fully_featured/setup.py deleted file mode 100644 index 8579370ae3c15..0000000000000 --- a/examples/project_fully_featured/setup.py +++ /dev/null @@ -1,42 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="project_fully_featured", - version="1!0+dev", - author="Dagster Labs", - author_email="hello@dagsterlabs.com", - classifiers=[ - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Operating System :: OS Independent", - ], - packages=find_packages(exclude=["test"]), - package_data={"project_fully_featured": ["hacker_news_dbt/*"]}, - install_requires=[ - "aiobotocore", - "dagster", - "dagster-aws", - "dagster-dbt", - "dagster-pandas", - "dagster-pyspark", - "dagster-slack", - "dagster-postgres", - "dbt-duckdb", - "dbt-snowflake", - "duckdb!=0.3.3, <= 6.0.0", # missing wheels - "mock", - "pandas", - "pyarrow>=4.0.0", - "pyspark", - "requests", - "gcsfs", - "fsspec", - "s3fs", - "scipy", - "scikit-learn", - "sqlalchemy!=1.4.42", # workaround for https://github.com/snowflakedb/snowflake-sqlalchemy/issues/350 - "snowflake-sqlalchemy<1.5.2", - ], - extras_require={"dev": ["dagster-webserver", "pytest"], "tests": ["mypy", "pylint", "pytest"]}, -) diff --git a/examples/quickstart_aws/pyproject.toml b/examples/quickstart_aws/pyproject.toml index 6e38c69ee30fb..2780274694bcf 100644 --- a/examples/quickstart_aws/pyproject.toml +++ b/examples/quickstart_aws/pyproject.toml @@ -1,3 +1,21 @@ +[project] +name = "quickstart_aws" +version = "0.1.0" +description = "Dagster project for AWS quickstart." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-aws", + "dagster-cloud", + "matplotlib", + "pandas", + "wordcloud", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +23,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "quickstart_aws.definitions" code_location_name = "quickstart_aws" + +[tool.setuptools.packages.find] +exclude = ["quickstart_aws_tests"] diff --git a/examples/quickstart_aws/setup.cfg b/examples/quickstart_aws/setup.cfg deleted file mode 100644 index 465d63ef15805..0000000000000 --- a/examples/quickstart_aws/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = quickstart_aws diff --git a/examples/quickstart_aws/setup.py b/examples/quickstart_aws/setup.py deleted file mode 100644 index b12a8cc21e289..0000000000000 --- a/examples/quickstart_aws/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="quickstart_aws", - packages=find_packages(exclude=["quickstart_aws_tests"]), - install_requires=[ - "dagster", - "dagster-aws", - "dagster-cloud", - "pandas", - "matplotlib", - "wordcloud", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/quickstart_etl/pyproject.toml b/examples/quickstart_etl/pyproject.toml index ef6a0222330b9..6ea5d376d6d0b 100644 --- a/examples/quickstart_etl/pyproject.toml +++ b/examples/quickstart_etl/pyproject.toml @@ -1,7 +1,27 @@ +[project] +name = "quickstart_etl" +version = "0.1.0" +description = "Dagster project for ETL quickstart." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "boto3", + "dagster", + "dagster-cloud", + "matplotlib", + "pandas", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "quickstart_etl.definitions" -code_location_name = "quickstart_etl" \ No newline at end of file +code_location_name = "quickstart_etl" + +[tool.setuptools.packages.find] +exclude = ["quickstart_etl_tests"] diff --git a/examples/quickstart_etl/setup.cfg b/examples/quickstart_etl/setup.cfg deleted file mode 100644 index c39bb92ddd8cc..0000000000000 --- a/examples/quickstart_etl/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = quickstart_etl diff --git a/examples/quickstart_etl/setup.py b/examples/quickstart_etl/setup.py deleted file mode 100644 index 30294c82bdb3c..0000000000000 --- a/examples/quickstart_etl/setup.py +++ /dev/null @@ -1,14 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="quickstart_etl", - packages=find_packages(exclude=["quickstart_etl_tests"]), - install_requires=[ - "dagster", - "dagster-cloud", - "boto3", - "pandas", - "matplotlib", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/quickstart_gcp/pyproject.toml b/examples/quickstart_gcp/pyproject.toml index 56cc233fe445a..92d652582f854 100644 --- a/examples/quickstart_gcp/pyproject.toml +++ b/examples/quickstart_gcp/pyproject.toml @@ -1,3 +1,27 @@ +[project] +name = "quickstart_gcp" +version = "0.1.0" +description = "Dagster project for GCP quickstart." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "boto3", # used by Dagster Cloud Serverless + "dagster", + "dagster-cloud", + "dagster-gcp", + "dagster-gcp-pandas", + "google-auth", + "matplotlib", + "pandas", + "pandas_gbq", + "textblob", + "tweepy", + "wordcloud", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +29,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "quickstart_gcp.definitions" code_location_name = "quickstart_gcp" + +[tool.setuptools.packages.find] +exclude = ["quickstart_gcp_tests"] diff --git a/examples/quickstart_gcp/setup.cfg b/examples/quickstart_gcp/setup.cfg deleted file mode 100644 index b540e43f29824..0000000000000 --- a/examples/quickstart_gcp/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = quickstart_gcp diff --git a/examples/quickstart_gcp/setup.py b/examples/quickstart_gcp/setup.py deleted file mode 100644 index 25600fe897cb1..0000000000000 --- a/examples/quickstart_gcp/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="quickstart_gcp", - packages=find_packages(exclude=["quickstart_gcp_tests"]), - install_requires=[ - "dagster", - "dagster-gcp", - "dagster-gcp-pandas", - "dagster-cloud", - "boto3", # used by Dagster Cloud Serverless - "pandas", - "matplotlib", - "textblob", - "tweepy", - "wordcloud", - "pandas_gbq", - "google-auth", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/quickstart_snowflake/pyproject.toml b/examples/quickstart_snowflake/pyproject.toml index 74778dda23b12..c6eaf677b48c0 100644 --- a/examples/quickstart_snowflake/pyproject.toml +++ b/examples/quickstart_snowflake/pyproject.toml @@ -1,7 +1,34 @@ +[project] +name = "quickstart_snowflake" +version = "0.1.0" +description = "Dagster project for Snowflake quickstart." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "boto3", + "dagster", + # snowflake-connector-python[pandas] is included by dagster-snowflake-pandas but it does + # not get included during pex dependency resolution, so we directly add this dependency + "dagster-cloud", + "dagster-snowflake-pandas", + "matplotlib", + "pandas", + "snowflake-connector-python[pandas]", + "textblob", + "tweepy", + "wordcloud", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "quickstart_snowflake.definitions" -code_location_name = "quickstart_snowflake" \ No newline at end of file +code_location_name = "quickstart_snowflake" + +[tool.setuptools.packages.find] +exclude = ["quickstart_snowflake_tests"] diff --git a/examples/quickstart_snowflake/setup.cfg b/examples/quickstart_snowflake/setup.cfg deleted file mode 100644 index ab64bc16dd441..0000000000000 --- a/examples/quickstart_snowflake/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = quickstart_snowflake diff --git a/examples/quickstart_snowflake/setup.py b/examples/quickstart_snowflake/setup.py deleted file mode 100644 index c24e6538b75d7..0000000000000 --- a/examples/quickstart_snowflake/setup.py +++ /dev/null @@ -1,21 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="quickstart_snowflake", - packages=find_packages(exclude=["quickstart_snowflake_tests"]), - install_requires=[ - "dagster", - # snowflake-connector-python[pandas] is included by dagster-snowflake-pandas but it does - # not get included during pex dependency resolution, so we directly add this dependency - "snowflake-connector-python[pandas]", - "dagster-snowflake-pandas", - "dagster-cloud", - "boto3", - "pandas", - "matplotlib", - "textblob", - "tweepy", - "wordcloud", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/tutorial/pyproject.toml b/examples/tutorial/pyproject.toml index 5594bcc4d06aa..e27509b024996 100644 --- a/examples/tutorial/pyproject.toml +++ b/examples/tutorial/pyproject.toml @@ -1,3 +1,23 @@ +[project] +name = "tutorial" +version = "0.1.0" +description = "Dagster project for tutorial." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster_duckdb_pandas", + "dagster_duckdb", + "dagster-cloud", + "Faker==18.4.0", + "matplotlib", + "pandas", + "requests", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +25,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "tutorial.definitions" code_location_name = "tutorial" + +[tool.setuptools.packages.find] +exclude = ["tutorial_tests"] diff --git a/examples/tutorial/setup.cfg b/examples/tutorial/setup.cfg deleted file mode 100644 index e79daae3cbeb5..0000000000000 --- a/examples/tutorial/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = tutorial diff --git a/examples/tutorial/setup.py b/examples/tutorial/setup.py deleted file mode 100644 index 864972f428e9d..0000000000000 --- a/examples/tutorial/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="tutorial", - packages=find_packages(exclude=["tutorial_tests"]), - install_requires=[ - "dagster", - "dagster_duckdb_pandas", - "dagster_duckdb", - "dagster-cloud", - "Faker==18.4.0", - "matplotlib", - "pandas", - "requests", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/tutorial_notebook_assets/pyproject.toml b/examples/tutorial_notebook_assets/pyproject.toml index fed528d4a7a14..57bc85d179c25 100644 --- a/examples/tutorial_notebook_assets/pyproject.toml +++ b/examples/tutorial_notebook_assets/pyproject.toml @@ -1,3 +1,24 @@ +[project] +name = "tutorial_notebook_assets" +version = "0.1.0" +description = "Dagster project for notebook assets tutorial." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagstermill", + "pandas", + "matplotlib", + "seaborn", + "scikit-learn", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +exclude = ["tutorial_notebook_assets"] diff --git a/examples/tutorial_notebook_assets/setup.cfg b/examples/tutorial_notebook_assets/setup.cfg deleted file mode 100644 index df0875e795399..0000000000000 --- a/examples/tutorial_notebook_assets/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = tutorial_notebook_assets diff --git a/examples/tutorial_notebook_assets/setup.py b/examples/tutorial_notebook_assets/setup.py deleted file mode 100644 index 91b4afc3c099a..0000000000000 --- a/examples/tutorial_notebook_assets/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="tutorial_notebook_assets", - packages=find_packages(exclude=["tutorial_notebook_assets"]), - install_requires=[ - "dagster", - "dagstermill", - "pandas", - "matplotlib", - "seaborn", - "scikit-learn", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/use_case_repository/pyproject.toml b/examples/use_case_repository/pyproject.toml index ec8cf7b72ca48..1c3f66e5b4ed8 100644 --- a/examples/use_case_repository/pyproject.toml +++ b/examples/use_case_repository/pyproject.toml @@ -1,7 +1,33 @@ +[project] +name = "use_case_repository" +version = "0.1.0" +description = "Example Dagster project demonstrating use case repository." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-aws", + "dagster-embedded-elt", + "dagster-pipes", + "dagster-snowflake", + "flask", + "markdown", + "polars", + "pymdown-extensions", + "python-frontmatter", + "sling", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest", "ruff"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "use_case_repository.definitions" -code_location_name = "use_case_repository" \ No newline at end of file +code_location_name = "use_case_repository" + +[tool.setuptools.packages.find] +exclude = ["use_case_repository_tests"] diff --git a/examples/use_case_repository/setup.cfg b/examples/use_case_repository/setup.cfg deleted file mode 100644 index eca5d2ef79c1d..0000000000000 --- a/examples/use_case_repository/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = use_case_repository diff --git a/examples/use_case_repository/setup.py b/examples/use_case_repository/setup.py deleted file mode 100644 index 339b21910d1e8..0000000000000 --- a/examples/use_case_repository/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="use_case_repository", - packages=find_packages(exclude=["use_case_repository_tests"]), - install_requires=[ - "dagster", - "dagster-aws", - "dagster-embedded-elt", - "dagster-pipes", - "dagster-snowflake", - "flask", - "markdown", - "polars", - "pymdown-extensions", - "python-frontmatter", - "sling", - ], - extras_require={ - "dev": [ - "dagster-webserver", - "pytest", - "ruff", - ] - }, -) diff --git a/examples/with_airflow/pyproject.toml b/examples/with_airflow/pyproject.toml index 74b11ab5207a9..9c9e74e18df57 100644 --- a/examples/with_airflow/pyproject.toml +++ b/examples/with_airflow/pyproject.toml @@ -1,6 +1,28 @@ +[project] +name = "with_great_expectations" +version = "0.1.0" +description = "Example Dagster project demonstrating integration with Great Expectations." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "apache-airflow", + "apache-airflow-providers-cncf-kubernetes>=4.4.0", # for the kubernetes operator + "apache-airflow-providers-docker>=3.1.0", + "dagster", + "dagster_airflow", + # Flask-session 0.6 is incompatible with certain airflow-provided test utilities. + "flask-session<0.6.0", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "with_airflow" + +[tool.setuptools.packages.find] +exclude = ["with_airflow_tests"] diff --git a/examples/with_airflow/setup.cfg b/examples/with_airflow/setup.cfg deleted file mode 100644 index 67f6296e8ca61..0000000000000 --- a/examples/with_airflow/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = with_airflow diff --git a/examples/with_airflow/setup.py b/examples/with_airflow/setup.py deleted file mode 100644 index cfb424ea541a3..0000000000000 --- a/examples/with_airflow/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="with_airflow", - packages=find_packages(exclude=["with_airflow_tests"]), - install_requires=[ - "dagster", - "dagster_airflow", - "apache-airflow", - # Flask-session 0.6 is incompatible with certain airflow-provided test - # utilities. - "flask-session<0.6.0", - # for the kubernetes operator - "apache-airflow-providers-cncf-kubernetes>=4.4.0", - "apache-airflow-providers-docker>=3.1.0", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/with_great_expectations/pyproject.toml b/examples/with_great_expectations/pyproject.toml index 2dfa9bf89b577..910665665d85c 100644 --- a/examples/with_great_expectations/pyproject.toml +++ b/examples/with_great_expectations/pyproject.toml @@ -1,7 +1,25 @@ +[project] +name = "with_great_expectations" +version = "0.1.0" +description = "Example Dagster project demonstrating integration with Great Expectations." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-ge", + "great_expectations>=0.14.12", # pinned because pip is using the cached wheel for 0.13.14 +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "with_great_expectations.definitions" -code_location_name = "with_great_expectations" \ No newline at end of file +code_location_name = "with_great_expectations" + +[tool.setuptools.packages.find] +exclude = ["with_great_expectations_tests"] diff --git a/examples/with_great_expectations/setup.cfg b/examples/with_great_expectations/setup.cfg deleted file mode 100644 index b67e29f62136e..0000000000000 --- a/examples/with_great_expectations/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = with_great_expectations diff --git a/examples/with_great_expectations/setup.py b/examples/with_great_expectations/setup.py deleted file mode 100644 index 2d70c4dd0aca1..0000000000000 --- a/examples/with_great_expectations/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="with_great_expectations", - packages=find_packages(exclude=["with_great_expectations_tests"]), - install_requires=[ - "dagster", - "dagster-ge", - "great_expectations>=0.14.12", # pinned because pip is using the cached wheel for 0.13.14 - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/with_openai/pyproject.toml b/examples/with_openai/pyproject.toml index 8de9a3bf6b59e..3aa5473cac70d 100644 --- a/examples/with_openai/pyproject.toml +++ b/examples/with_openai/pyproject.toml @@ -1,3 +1,23 @@ +[project] +name = "with_openai" +version = "0.1.0" +description = "Example Dagster project demonstrating Open AI." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-aws", + "dagster-openai", + "faiss-cpu==1.8.0", + "filelock", + "langchain==0.2.10", + "langchain-community==0.2.9", + "langchain-openai==0.1.14", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -5,3 +25,6 @@ build-backend = "setuptools.build_meta" [tool.dagster] module_name = "with_openai.definitions" code_location_name = "with_openai" + +[tool.setuptools.packages.find] +exclude = ["with_openai_tests"] diff --git a/examples/with_openai/setup.cfg b/examples/with_openai/setup.cfg deleted file mode 100644 index 34db98d7dc92f..0000000000000 --- a/examples/with_openai/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = with_openai diff --git a/examples/with_openai/setup.py b/examples/with_openai/setup.py deleted file mode 100644 index 814295ee34728..0000000000000 --- a/examples/with_openai/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="with_openai", - packages=find_packages(exclude=["with_openai_tests"]), - install_requires=[ - "dagster", - "dagster-aws", - "dagster-openai", - "faiss-cpu==1.8.0", - "filelock", - "langchain==0.2.10", - "langchain-community==0.2.9", - "langchain-openai==0.1.14", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/with_pyspark/pyproject.toml b/examples/with_pyspark/pyproject.toml index 4a081c26e26ed..d1353cb4f4406 100644 --- a/examples/with_pyspark/pyproject.toml +++ b/examples/with_pyspark/pyproject.toml @@ -1,6 +1,24 @@ +[project] +name = "with_pyspark" +version = "0.1.0" +description = "Example Dagster project demonstrating PySpark." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-spark", + "dagster-pyspark", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "with_pyspark" + +[tool.setuptools.packages.find] +exclude = ["with_pyspark_tests"] diff --git a/examples/with_pyspark/setup.cfg b/examples/with_pyspark/setup.cfg deleted file mode 100644 index f28d76572e1bd..0000000000000 --- a/examples/with_pyspark/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = with_pyspark diff --git a/examples/with_pyspark/setup.py b/examples/with_pyspark/setup.py deleted file mode 100644 index 421c5308d19b7..0000000000000 --- a/examples/with_pyspark/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="with_pyspark", - packages=find_packages(exclude=["with_pyspark_tests"]), - install_requires=[ - "dagster", - "dagster-spark", - "dagster-pyspark", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/with_pyspark_emr/pyproject.toml b/examples/with_pyspark_emr/pyproject.toml index 04749ba81dd34..95011a1528272 100644 --- a/examples/with_pyspark_emr/pyproject.toml +++ b/examples/with_pyspark_emr/pyproject.toml @@ -1,6 +1,24 @@ +[project] +name = "with_pyspark_emr" +version = "0.1.0" +description = "Example Dagster project demonstrating PySpark EMR." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-aws", + "dagster-pyspark", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] -module_name = "with_pyspark_emr" \ No newline at end of file +module_name = "with_pyspark_emr" + +[tool.setuptools.packages.find] +exclude = ["with_pyspark_emr_tests"] diff --git a/examples/with_pyspark_emr/setup.cfg b/examples/with_pyspark_emr/setup.cfg deleted file mode 100644 index 528bc9f3b8d21..0000000000000 --- a/examples/with_pyspark_emr/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = with_pyspark_emr diff --git a/examples/with_pyspark_emr/setup.py b/examples/with_pyspark_emr/setup.py deleted file mode 100644 index 84c19c8033424..0000000000000 --- a/examples/with_pyspark_emr/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="with_pyspark_emr", - packages=find_packages(exclude=["with_pyspark_emr_tests"]), - install_requires=[ - "dagster", - "dagster-aws", - "dagster-pyspark", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) diff --git a/examples/with_wandb/pyproject.toml b/examples/with_wandb/pyproject.toml index e6d397b503f5a..1b9c7b40f0106 100644 --- a/examples/with_wandb/pyproject.toml +++ b/examples/with_wandb/pyproject.toml @@ -1,7 +1,30 @@ +[project] +name = "with_wandb" +version = "0.1.0" +description = "Example Dagster project demonstrating wandb." +readme = "README.md" +requires-python = ">=3.8,<3.13" +dependencies = [ + "dagster", + "dagster-wandb", + "joblib", + "onnx>=1.13.0", # Ensure a version is installed that is protobuf 4 compatible + "onnxruntime", + "skl2onnx", + "torch", + "torchvision", +] + +[project.optional-dependencies] +dev = ["dagster-webserver", "pytest"] + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.dagster] module_name = "with_wandb.definitions" -code_location_name = "with_wandb" \ No newline at end of file +code_location_name = "with_wandb" + +[tool.setuptools.packages.find] +exclude = ["with_wandb_tests"] diff --git a/examples/with_wandb/setup.cfg b/examples/with_wandb/setup.cfg deleted file mode 100644 index 701d23a1fe56a..0000000000000 --- a/examples/with_wandb/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = with_wandb diff --git a/examples/with_wandb/setup.py b/examples/with_wandb/setup.py deleted file mode 100644 index 6c6419ba0e9de..0000000000000 --- a/examples/with_wandb/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="with_wandb", - packages=find_packages(exclude=["with_wandb_tests"]), - install_requires=[ - "dagster", - "dagster-wandb", - "onnxruntime", - "skl2onnx", - "onnx>=1.13.0", # Ensure a version is installed that is protobuf 4 compatible - "joblib", - "torch", - "torchvision", - ], - extras_require={"dev": ["dagster-webserver", "pytest"]}, -) From d2ec720ebf20df80f473e330d34693a4ec458788 Mon Sep 17 00:00:00 2001 From: Daniel Bartley Date: Sat, 19 Oct 2024 00:32:48 +1100 Subject: [PATCH 3/3] docs: consolidate setup config to pyproject.toml --- .../dagster/recommended-project-structure.mdx | 2 +- .../understanding-dagster-project-files.mdx | 33 +++++++------------ 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/docs/content/guides/dagster/recommended-project-structure.mdx b/docs/content/guides/dagster/recommended-project-structure.mdx index e703426ae7c5a..ee7c23be291da 100644 --- a/docs/content/guides/dagster/recommended-project-structure.mdx +++ b/docs/content/guides/dagster/recommended-project-structure.mdx @@ -162,7 +162,7 @@ We recommend defining ops and graphs a job file along with the job definition wi ## For multiple code locations -So far, we've discussed our recommendations for structuring a large project which contains only one code location. Dagster also allows you to structure a project with multiple definitions. We don't recommend over-abstracting too early; in most cases, one code location should be sufficient. A helpful pattern uses multiple code locations to separate conflicting dependencies, where each definition has its own package requirements (e.g., `setup.py`) and deployment specs (e.g., Dockerfile). +So far, we've discussed our recommendations for structuring a large project which contains only one code location. Dagster also allows you to structure a project with multiple definitions. We don't recommend over-abstracting too early; in most cases, one code location should be sufficient. A helpful pattern uses multiple code locations to separate conflicting dependencies, where each definition has its own package requirements (e.g., `pyproject.toml`) and deployment specs (e.g., Dockerfile). To include multiple code locations in a single project, you'll need to add a [configuration file](/guides/understanding-dagster-project-files#configuration-files) to your project: diff --git a/docs/content/guides/understanding-dagster-project-files.mdx b/docs/content/guides/understanding-dagster-project-files.mdx index 22e0718cb6f49..23b9ef43b9480 100644 --- a/docs/content/guides/understanding-dagster-project-files.mdx +++ b/docs/content/guides/understanding-dagster-project-files.mdx @@ -131,8 +131,8 @@ Let's take a look at what each of these files and directories does: pyproject.toml - A file that specifies package core metadata in a static, tool-agnostic - way. + A file that specifies package configuration in a static, tool-agnostic + format.

This file includes a tool.dagster section which references the @@ -148,16 +148,10 @@ Let's take a look at what each of these files and directories does: PEP-518 {" "} - and meant to replace setup.py, but we may still include a{" "} - setup.py for compatibility with tools that do not use this + and meant to replace setup.py, but we may still include {" "} + setup.cfg and {" "} + setup.py files for compatibility with tools that do not use this spec. - - - - setup.py - - A build script with Python package dependencies for your new project as - a package. Use this file to specify dependencies.

Note: If using Dagster+, add dagster-cloud as a dependency. @@ -169,6 +163,13 @@ Let's take a look at what each of these files and directories does: commands. + + setup.py + + A optional build script for your new project that specifies Python package dependencies + and project setup. Some developers and tooling may work best with this file. + + @@ -301,8 +302,6 @@ For local development, a project with a single code location might look like thi ├── my_dagster_project_tests ├── dagster.yaml ## optional, used for instance settings ├── pyproject.toml ## optional, used to define the project as a module -├── setup.cfg -├── setup.py └── tox.ini ``` @@ -323,8 +322,6 @@ For local development, a project with multiple code locations might look like th ├── my_dagster_project_tests ├── dagster.yaml ## optional, used for instance settings ├── pyproject.toml -├── setup.cfg -├── setup.py ├── tox.ini └── workspace.yaml ## defines multiple code locations ``` @@ -350,8 +347,6 @@ A Dagster project deployed to your infrastructure might look like this: ├── my_dagster_project_tests ├── dagster.yaml ## optional, used for instance settings ├── pyproject.toml -├── setup.cfg -├── setup.py ├── tox.ini └── workspace.yaml ## defines multiple code locations ``` @@ -378,8 +373,6 @@ For a Dagster+ Serverless deployment, a project might look like this: ├── dagster_cloud.yaml ## defines code locations ├── deployment_settings.yaml ## optional, defines settings for full deployments ├── pyproject.toml -├── setup.cfg -├── setup.py └── tox.ini ``` @@ -402,8 +395,6 @@ For a Dagster+ Hybrid deployment, a project might look like this: ├── dagster_cloud.yaml ## defines code locations ├── deployment_settings.yaml ## optional, defines settings for full deployments ├── pyproject.toml -├── setup.cfg -├── setup.py └── tox.ini ```