diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py index 8355164986d97..c2b2081038f96 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py @@ -367,9 +367,6 @@ def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: PackageSpec( "examples/experimental/dagster-blueprints", ), - PackageSpec( - "examples/experimental/dagster-airlift", - ), # Runs against live dbt cloud instance, we only want to run on commits and on the # nightly build PackageSpec( @@ -393,18 +390,10 @@ def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]: timeout_in_minutes=30, queue=BuildkiteQueue.DOCKER, ), - PackageSpec( - "examples/experimental/dagster-airlift/perf-harness", - always_run_if=has_dagster_airlift_changes, - ), PackageSpec( "examples/airlift-migration-tutorial", always_run_if=has_dagster_airlift_changes, ), - PackageSpec( - "examples/experimental/dagster-airlift/kitchen-sink", - always_run_if=has_dagster_airlift_changes, - ), PackageSpec( "examples/experimental/dagster-dlift", name="dlift", @@ -748,6 +737,14 @@ def tox_factors_for_folder(tests_folder_name: str) -> List[str]: AvailablePythonVersion.V3_12, ], ), + PackageSpec( + "python_modules/libraries/dagster-airlift/perf-harness", + always_run_if=has_dagster_airlift_changes, + ), + PackageSpec( + "python_modules/libraries/dagster-airlift/kitchen-sink", + always_run_if=has_dagster_airlift_changes, + ), PackageSpec( ".buildkite/dagster-buildkite", run_pytest=False, diff --git a/docs/content/integrations/airlift/federation-tutorial/setup.mdx b/docs/content/integrations/airlift/federation-tutorial/setup.mdx index 320f640f89b13..bcc250c834f0f 100644 --- a/docs/content/integrations/airlift/federation-tutorial/setup.mdx +++ b/docs/content/integrations/airlift/federation-tutorial/setup.mdx @@ -8,19 +8,39 @@ In this step, we'll ## Installation & Project Structure -First, clone the tutorial example repo locally, and enter the repo directory. +First, we'll create a fresh virtual environment using `uv` and activate it. ```bash -git clone git@github.com:dagster-io/airlift-migration-tutorial.git -cd airlift-federation-tutorial +pip install uv +uv venv +source .venv/bin/activate ``` -Next, we'll create a fresh virtual environment using `uv`. +Next, we'll install Dagster, and verify that the dagster CLI is available. ```bash -pip install uv -uv venv -source .venv/bin/activate +uv pip install dagster +dagster --version +``` + +Finally, we'll install the tutorial example code. + +```bash +dagster project from-example --name airlift-federation-tutorial --example airlift-federation-tutorial +``` + +### Project Structure + +The following explains the structure of the repo. + +```plaintext +airlift_federation_tutorial +├── constants.py: Contains constant values used throughout both Airflow and Dagster +├── dagster_defs: Contains Dagster definitions +│ ├── definitions.py: Empty starter file for following along with the tutorial +│ └── stages: Contains reference implementations for each stage of the migration process. +├── metrics_airflow_dags: Contains the Airflow DAGs for the "downstream" airflow instance +└── warehouse_airflow_dags: Contains the Airflow DAGs for the "upstream" airflow instance ``` ## Running Airflow locally diff --git a/docs/content/integrations/airlift/tutorial/setup.mdx b/docs/content/integrations/airlift/tutorial/setup.mdx index f2291e839e7cd..b39a9b31c5aeb 100644 --- a/docs/content/integrations/airlift/tutorial/setup.mdx +++ b/docs/content/integrations/airlift/tutorial/setup.mdx @@ -8,19 +8,42 @@ In this step, we'll ## Installation & Project Structure -First, clone the tutorial example repo locally, and enter the repo directory. +First, we'll create a fresh virtual environment using `uv`. ```bash -git clone git@github.com:dagster-io/airlift-migration-tutorial.git -cd airlift-migration-tutorial +pip install uv +uv venv +source .venv/bin/activate ``` -Next, we'll create a fresh virtual environment using `uv`. +Next, we'll install Dagster, and verify that the dagster CLI is available. ```bash -pip install uv -uv venv -source .venv/bin/activate +uv pip install dagster +dagster --version +``` + +First, we'll create a fresh virtual environment using `uv`. + +```bash +dagster project from-example --name airlift-migration-tutorial --example airlift-migration-tutorial +``` + +### Project Structure + +The following explains the structure of the repo. + +```plaintext +tutorial_example +├── shared: Contains shared Python & SQL code used Airflow and proxied Dagster code +│ +├── dagster_defs: Contains Dagster definitions +│ ├── stages: Contains reference implementations of each stage of the migration process +│ ├── definitions.py: Empty starter file for following along with the tutorial +│ +├── airflow_dags: Contains the Airflow DAG and associated files +│ ├── proxied_state: Contains migration state files for each DAG, see migration step below +│ ├── dags.py: The Airflow DAG definition ``` ## Running Airflow locally diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 436989ae0d65d..26e5dc2a1cbfb 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -14,7 +14,6 @@ import sys ignored_folders = ["dagster-test"] -additional_folders = ["../../examples/experimental/dagster-airlift"] base_path = "../../python_modules" python_modules_path = os.path.abspath(base_path) libraries_path = os.path.abspath(os.path.join(base_path, "libraries")) @@ -35,8 +34,6 @@ paths.append(folder_path) # Add the _ext folder paths.append(os.path.abspath("./_ext")) -for path in additional_folders: - paths.append(os.path.abspath(path)) for path in paths: sys.path.insert(0, path) diff --git a/docs/tox.ini b/docs/tox.ini index 246cc13f0df99..e5468066969f6 100644 --- a/docs/tox.ini +++ b/docs/tox.ini @@ -48,7 +48,7 @@ deps = -e ../python_modules/libraries/dagster-sigma -e ../python_modules/libraries/dagster-tableau -e ../python_modules/libraries/dagster-powerbi - -e ../examples/experimental/dagster-airlift[tutorial,core,in-airflow,dbt] + -e ../python_modules/libraries/dagster-airlift[tutorial,core,in-airflow,dbt] -e ../examples/airlift-migration-tutorial sling diff --git a/examples/airlift-federation-tutorial/README.md b/examples/airlift-federation-tutorial/README.md index 17a6cf4d6db87..06aff3f336e88 100644 --- a/examples/airlift-federation-tutorial/README.md +++ b/examples/airlift-federation-tutorial/README.md @@ -1,6 +1,6 @@ ## Airlift Federation Tutorial Code -This repo is the code that the Airlift federation tutorial is based off of. Follow along on the main docs site [here](https://docs.dagster.io/integrations/airlift). +This is the code that the Airlift federation tutorial is based off of. This example demonstrates how to use the `dagster-airlift` package to unify multiple Airflow instances into Dagster as a single control plane, and then federate execution between those Airflow instances. diff --git a/examples/airlift-federation-tutorial/tox.ini b/examples/airlift-federation-tutorial/tox.ini index e81bd74be122f..32185a93e2736 100644 --- a/examples/airlift-federation-tutorial/tox.ini +++ b/examples/airlift-federation-tutorial/tox.ini @@ -15,7 +15,7 @@ deps = -e ../../python_modules/dagster-pipes -e ../../python_modules/dagster-graphql -e ../../python_modules/libraries/dagster-dbt - -e ../experimental/dagster-airlift[core,dbt,test,in-airflow] + -e ../../python_modules/libraries/dagster-airlift[core,dbt,test,in-airflow] -e . pandas allowlist_externals = diff --git a/examples/airlift-migration-tutorial/tox.ini b/examples/airlift-migration-tutorial/tox.ini index d6b8ca8fb3479..4ea1a85623347 100644 --- a/examples/airlift-migration-tutorial/tox.ini +++ b/examples/airlift-migration-tutorial/tox.ini @@ -15,7 +15,7 @@ deps = -e ../../python_modules/dagster-pipes -e ../../python_modules/dagster-graphql -e ../../python_modules/libraries/dagster-dbt - -e ../experimental/dagster-airlift[core,dbt,test,in-airflow] + -e ../../python_modules/libraries/dagster-airlift[core,dbt,test,in-airflow] -e . pandas allowlist_externals = diff --git a/examples/experimental/dagster-airlift/CHANGES.md b/examples/experimental/dagster-airlift/CHANGES.md deleted file mode 100644 index a8f6f1272a3ee..0000000000000 --- a/examples/experimental/dagster-airlift/CHANGES.md +++ /dev/null @@ -1,76 +0,0 @@ -# Changelog - -## 0.0.27 - -### New - -- Airlift CLI: You can now proxy airlift state in your airflow directory by calling `dagster-airlift proxy scaffold` from within your Airflow codebase (airflow must be installed, and `AIRFLOW_HOME` must be set) -- `dagster-airlift[in-airflow]` no longer has a direct pin on airflow itself. Instead we rely on users installing their own 2.0.0 or greater version of Airflow. -- Dag-level mapping has been introduced. Using the `assets_with_dag_mappings` API, you can map individual assets to an entire DAG, and those assets will receive materializations on successful runs of that DAG. See the tutorial section on dag-level mapping for more. -- Python 3.12 is now explicitly supported. -- Time-windowed partitioned assets mapped to Airflow will now automatically receive partitioned materializations corresponding to the `execution_date` in airflow. See the tutorial section on partitioning for more. - -### Tutorial - -- Tutorial section on dealing with changing airflow -- Tutorial section on partitioning -- Tutorial section on dag-level mapping -- The tutorial has been changed to use `assets_with_task_mappings` instead of `task_defs` to maintain a consistent API across integrations. - -### Breaking Changes - -- Proxy operators now makes the assumption that all assets mapped to a given airflow task must be launchable within a single run (this works out of the box on Dagster 1.8 or greater). -- The `dagster_operator_klass` method has been removed, and the base class has changed. Instead of overriding `BaseProxyToDagsterOperator`, users will now subclass `BaseProxyTaskToDagsterOperator`, and use the `build_from_task_fn` argument: - -```python -proxying_to_dagster( - ..., - # This method is implemented by default, but can be overridden. - build_from_task_fn=MyCustomOperator.build_from_task, -) -``` - -## 0.0.26 - -### New - -- Add python 3.8 support -- `BaseMaterializeAssetsOperator` for kicking off execution of a discrete set of Dagster assets from Airflow. - -## 0.0.25 - -### Breaking Changes - -- `dagster-airlift[core,mwaa,dbt]` is now pinned on Dagster >= 1.8.10. - -## 0.0.24 - -### New - -- Rebuilt `build_defs_from_airflow_instance` on top of lazy-loading Definitions abstractions. -- The initial run of `build_defs_from_airflow_instance` has been made much faster by batching retrieval of task info per dag. -- A new `targeted_by_multiple_tasks` function in `dagster_airlift.core` which allows for targeting the same set of assets from tasks in multiple dags. See the docstring for more. - -### Breaking Changes - -- `dagster-airlift[core,mwaa,dbt]` is now pinned on Dagster >= 1.8.8. -- `migrating` terminology has been updated to `proxied` terminology. - - `mark_as_dagster_migrating` has been renamed to `proxying_to_dagster` - - In examples, the `migration_state` directories have been renamed to `proxied_state`. - - In each `yaml` file in the proxied directory, the `migrated` bit has been renamed to the `proxied` bit. So, if prior your migration file looked like this: - -```yaml -# migration_state/my_dag.yaml -tasks: - id: foo - migrated: False -``` - - it should instead look like this: - -```yaml -# proxied_state/my_dag.yaml -tasks: - id: foo - proxied: False -``` diff --git a/examples/experimental/dagster-airlift/MANIFEST.in b/examples/experimental/dagster-airlift/MANIFEST.in deleted file mode 100644 index d71fef85e5369..0000000000000 --- a/examples/experimental/dagster-airlift/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include README.md -include LICENSE -include dagster_airlift/py.typed \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/Makefile b/examples/experimental/dagster-airlift/Makefile deleted file mode 100644 index 1f570472a0159..0000000000000 --- a/examples/experimental/dagster-airlift/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -# Makefile - -# Uploads the package to PyPI after bumping the latest version in setup.py. -adhoc_pypi: - @chmod +x scripts/increment_pypi_version.sh - @./scripts/increment_pypi_version.sh - -# Mirrors the tutorial to the dagster-airlift-migration-tutorial repo: https://github.com/dagster-io/airlift-migration-tutorial -mirror_tutorial: - @chmod +x scripts/mirror_tutorial.sh - @./scripts/mirror_tutorial.sh - -mirror_federation_tutorial: - @chmod +x scripts/mirror_federation_tutorial.sh - @./scripts/mirror_federation_tutorial.sh - -# Runs the full release process for dagster-airlift. -# - Enforces that we're on master -# - Bumps the version in setup.py -# - Pushes the changes to PyPI -# - Mirrors the tutorial to the dagster-airlift-migration-tutorial repo -# - Creates a release branch -# - Pushes the release branch to origin -adhoc_release: - @if [ "$$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then \ - echo "Error: Not on master branch. Please switch to master before running this command."; \ - exit 1; \ - fi - @echo "Releasing dagster-airlift..." - @make adhoc_pypi - @echo "Mirroring tutorial..." - @make mirror_tutorial - @echo "Mirroring federation tutorial..." - @make mirror_federation_tutorial - @git add . - @git checkout -b airlift-$$(./scripts/extract_pypi_version.sh) - @git commit -m "[dagster-airlift] $$(./scripts/extract_pypi_version.sh)" - @git push origin airlift-$$(./scripts/extract_pypi_version.sh) - echo "Airlift release branch created. Please create a PR and merge it to master." - @echo "Returning to master branch..." - @git checkout master - -find_airlift_commits: - @echo "Finding commits for dagster-airlift..." - @chmod +x scripts/find_airlift_commits.sh - @./scripts/find_airlift_commits.sh \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/dagster_airlift/dbt/__init__.py b/examples/experimental/dagster-airlift/dagster_airlift/dbt/__init__.py deleted file mode 100644 index 326d4a6fd5921..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift/dbt/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .multi_asset import dbt_defs as dbt_defs diff --git a/examples/experimental/dagster-airlift/dagster_airlift/k8s/__init__.py b/examples/experimental/dagster-airlift/dagster_airlift/k8s/__init__.py deleted file mode 100644 index 607336d7ecd81..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift/k8s/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .asset import k8s_pod_defs as k8s_pod_defs diff --git a/examples/experimental/dagster-airlift/dagster_airlift/mwaa/__init__.py b/examples/experimental/dagster-airlift/dagster_airlift/mwaa/__init__.py deleted file mode 100644 index a382b20ed8100..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift/mwaa/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .auth import MwaaSessionAuthBackend as MwaaSessionAuthBackend diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/extra_key/extra_key.yaml b/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/extra_key/extra_key.yaml deleted file mode 100644 index dcf3d144cca9c..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/extra_key/extra_key.yaml +++ /dev/null @@ -1,4 +0,0 @@ -tasks: - - id: some_task - proxied: True - extra_key: extra_value \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/nonsense/nonsense_value.yaml b/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/nonsense/nonsense_value.yaml deleted file mode 100644 index fd2ac3e4eff9d..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/nonsense/nonsense_value.yaml +++ /dev/null @@ -1,3 +0,0 @@ -tasks: - - id: some_task - proxied: elephant \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/first.yaml b/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/first.yaml deleted file mode 100644 index 00752aa3fa81b..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/first.yaml +++ /dev/null @@ -1,7 +0,0 @@ -tasks: - - id: first_task - proxied: True - - id: second_task - proxied: False - - id: third_task - proxied: True \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/second.yaml b/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/second.yaml deleted file mode 100644 index 657bb74e1e5c9..0000000000000 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/second.yaml +++ /dev/null @@ -1,5 +0,0 @@ -tasks: - - id: some_task - proxied: True - - id: other_task - proxied: False \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/examples/tutorial-example/customers.csv b/examples/experimental/dagster-airlift/examples/tutorial-example/customers.csv deleted file mode 100644 index d04d951e7e466..0000000000000 --- a/examples/experimental/dagster-airlift/examples/tutorial-example/customers.csv +++ /dev/null @@ -1,101 +0,0 @@ -customer_id,first_name,last_name,first_order,most_recent_order,number_of_orders,customer_lifetime_value -1,Michael,P.,2018-01-01,2018-02-10,2.0,33.0 -2,Shawn,M.,2018-01-11,2018-01-11,1.0,23.0 -3,Kathleen,P.,2018-01-02,2018-03-11,3.0,65.0 -6,Sarah,R.,2018-02-19,2018-02-19,1.0,8.0 -7,Martin,M.,2018-01-14,2018-01-14,1.0,26.0 -8,Frank,R.,2018-01-29,2018-03-12,2.0,45.0 -9,Jennifer,F.,2018-03-17,2018-03-17,1.0,30.0 -11,Fred,S.,2018-03-23,2018-03-23,1.0,3.0 -12,Amy,D.,2018-03-03,2018-03-03,1.0,4.0 -13,Kathleen,M.,2018-03-07,2018-03-07,1.0,26.0 -16,Amanda,H.,2018-02-02,2018-02-02,1.0,12.0 -18,Johnny,K.,2018-02-27,2018-02-27,1.0,29.0 -19,Virginia,F.,2018-03-16,2018-03-16,1.0,3.0 -20,Anna,A.,2018-01-23,2018-01-23,1.0,15.0 -21,Willie,H.,2018-03-28,2018-03-28,1.0,22.0 -22,Sean,H.,2018-01-26,2018-03-01,3.0,52.0 -25,Victor,H.,2018-01-17,2018-03-20,2.0,24.0 -26,Aaron,R.,2018-02-11,2018-03-08,2.0,8.0 -27,Benjamin,B.,2018-02-21,2018-04-04,2.0,27.0 -28,Lisa,W.,2018-02-04,2018-02-04,1.0,3.0 -30,Christina,W.,2018-03-02,2018-03-14,2.0,57.0 -31,Jane,G.,2018-02-17,2018-02-17,1.0,18.0 -32,Thomas,O.,2018-01-28,2018-01-28,1.0,30.0 -33,Katherine,M.,2018-02-13,2018-02-13,1.0,14.0 -34,Jennifer,S.,2018-02-26,2018-02-26,1.0,3.0 -35,Sara,T.,2018-02-21,2018-03-21,2.0,34.0 -36,Harold,O.,2018-03-10,2018-03-10,1.0,28.0 -38,Dennis,J.,2018-02-06,2018-02-06,1.0,15.0 -39,Louise,W.,2018-01-18,2018-01-18,1.0,10.0 -40,Maria,A.,2018-01-17,2018-01-17,1.0,3.0 -41,Gloria,C.,2018-04-07,2018-04-07,1.0,10.0 -42,Diana,S.,2018-02-04,2018-03-12,2.0,27.0 -46,Norma,C.,2018-03-24,2018-03-27,2.0,64.0 -47,Marie,P.,2018-03-26,2018-03-31,2.0,36.0 -51,Howard,R.,2018-01-28,2018-02-23,3.0,99.0 -52,Laura,F.,2018-03-23,2018-03-23,1.0,27.0 -53,Anne,B.,2018-01-12,2018-03-11,2.0,39.0 -54,Rose,M.,2018-01-07,2018-03-24,5.0,57.0 -57,Paul,W.,2018-01-31,2018-03-05,2.0,26.0 -59,Adam,A.,2018-01-15,2018-01-15,1.0,1.0 -63,Edward,G.,2018-03-03,2018-04-03,2.0,23.0 -64,David,C.,2018-01-05,2018-01-20,2.0,30.0 -66,Adam,W.,2018-02-17,2018-04-03,3.0,39.0 -68,Jesse,E.,2018-03-26,2018-03-26,1.0,23.0 -69,Janet,P.,2018-02-02,2018-03-18,2.0,32.0 -70,Helen,F.,2018-03-06,2018-03-26,2.0,54.0 -71,Gerald,C.,2018-01-18,2018-02-24,3.0,44.0 -76,Barbara,W.,2018-03-23,2018-03-23,1.0,2.0 -79,Jack,R.,2018-02-28,2018-03-11,2.0,27.0 -80,Phillip,H.,2018-02-08,2018-02-08,1.0,29.0 -84,Christina,R.,2018-01-17,2018-04-02,2.0,36.0 -85,Theresa,M.,2018-02-10,2018-04-09,2.0,33.0 -86,Jason,C.,2018-01-24,2018-01-24,1.0,8.0 -88,Adam,T.,2018-01-09,2018-01-09,1.0,16.0 -89,Margaret,J.,2018-04-07,2018-04-07,1.0,14.0 -90,Paul,P.,2018-03-23,2018-04-06,2.0,43.0 -91,Todd,W.,2018-03-27,2018-03-27,1.0,29.0 -92,Willie,O.,2018-02-16,2018-02-16,1.0,17.0 -93,Frances,R.,2018-03-01,2018-03-01,1.0,2.0 -94,Gregory,H.,2018-01-04,2018-01-29,2.0,24.0 -99,Mary,G.,2018-01-14,2018-02-14,2.0,44.0 -50,Billy,L.,2018-01-05,2018-02-20,2.0,47.0 -4,Jimmy,C.,,,, -5,Katherine,R.,,,, -10,Henry,W.,,,, -14,Steve,F.,,,, -15,Teresa,H.,,,, -17,Kimberly,R.,,,, -23,Mildred,A.,,,, -24,David,G.,,,, -29,Benjamin,K.,,,, -37,Shirley,J.,,,, -43,Kelly,N.,,,, -44,Jane,R.,,,, -45,Scott,B.,,,, -48,Lillian,C.,,,, -49,Judy,N.,,,, -55,Nicholas,R.,,,, -56,Joshua,K.,,,, -58,Kathryn,K.,,,, -60,Norma,W.,,,, -61,Timothy,R.,,,, -62,Elizabeth,P.,,,, -65,Brenda,W.,,,, -67,Michael,H.,,,, -72,Kathryn,O.,,,, -73,Alan,B.,,,, -74,Harry,A.,,,, -75,Andrea,H.,,,, -77,Anne,W.,,,, -78,Harry,H.,,,, -81,Shirley,H.,,,, -82,Arthur,D.,,,, -83,Virginia,R.,,,, -87,Phillip,B.,,,, -95,Lisa,P.,,,, -96,Jacqueline,A.,,,, -97,Shirley,D.,,,, -98,Nicole,M.,,,, -100,Jean,M.,,,, diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml b/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml deleted file mode 100644 index 03068f4fcf047..0000000000000 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml +++ /dev/null @@ -1,3 +0,0 @@ -tasks: - - id: task - proxied: False \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml b/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml deleted file mode 100644 index 03068f4fcf047..0000000000000 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml +++ /dev/null @@ -1,3 +0,0 @@ -tasks: - - id: task - proxied: False \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/migrated_daily_interval_dag.yaml b/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/migrated_daily_interval_dag.yaml deleted file mode 100644 index 4b668df8dda47..0000000000000 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/migrated_daily_interval_dag.yaml +++ /dev/null @@ -1,3 +0,0 @@ -tasks: - - id: my_task - proxied: True \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag.yaml b/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag.yaml deleted file mode 100644 index 25a5a21ccae89..0000000000000 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag.yaml +++ /dev/null @@ -1 +0,0 @@ -proxied: True \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag_custom_callback.yaml b/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag_custom_callback.yaml deleted file mode 100644 index 25a5a21ccae89..0000000000000 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag_custom_callback.yaml +++ /dev/null @@ -1 +0,0 @@ -proxied: True \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/migration_state/dag_0.yaml b/examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/migration_state/dag_0.yaml deleted file mode 100644 index c34d6f2702670..0000000000000 --- a/examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/migration_state/dag_0.yaml +++ /dev/null @@ -1,3 +0,0 @@ -tasks: -- id: task_0_0 - proxied: false diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/__init__.py b/examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/examples/experimental/dagster-airlift/scripts/build_and_publish.sh b/examples/experimental/dagster-airlift/scripts/build_and_publish.sh deleted file mode 100755 index e3202023f051a..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/build_and_publish.sh +++ /dev/null @@ -1,68 +0,0 @@ -# How to release: -# 2. ensure you have an API key from the elementl PyPI account (account is in the password manager) -# 3. run make adhoc_pypi from the root of the dagster-airlift directory -# 4. once propted, use '__token__' for the username and the API key for the password - -# Define the path to the .pypirc file -PYPIRC_FILE="$HOME/.pypirc" - -# Define cleanup function -cleanup() { - echo "Cleaning up..." - rm -rf dist/* - rm -rf dagster_airlift_prerelease/dagster_airlift_patched -} - -# Set trap to call cleanup function on script exit -trap cleanup EXIT - -# Check if the .pypirc file exists -if [ ! -f "$PYPIRC_FILE" ]; then - echo ".pypirc file not found in $HOME." - - # Prompt the user for the API token - read -p "Enter your API token (must start with 'pypi-'): " API_TOKEN - - # Check if the API token starts with 'pypi-' - if [[ $API_TOKEN != pypi-* ]]; then - echo "Invalid API token. It must start with 'pypi-'." - exit 1 - fi - - # Create the .pypirc file and write the configuration - cat < "$PYPIRC_FILE" -[pypi] -username = __token__ -password = $API_TOKEN -EOF - - echo ".pypirc file created successfully." -else - echo ".pypirc file already exists in $HOME. Using that as pypi credentials." -fi - -rm -rf dist/* -rm -rf dagster_airlift_prerelease/dagster_airlift_patched -mkdir -p dagster_airlift_prerelease/dagster_airlift_patched -cp -R dagster_airlift dagster_airlift_prerelease/dagster_airlift_patched -echo "Building package..." -python3 -m build -echo "Uploading to pypi..." -# Capture the output of the twine upload command -TWINE_OUTPUT=$(python3 -m twine upload --repository pypi dist/* --verbose 2>&1) -TWINE_EXIT_CODE=$? - -# Check if the output contains a 400 error -if echo "$TWINE_OUTPUT" | grep -q "400 Bad Request"; then - echo "Error: Twine upload failed with a 400 Bad Request error." - echo "Twine output:" - echo "$TWINE_OUTPUT" - exit 1 -elif [ $TWINE_EXIT_CODE -ne 0 ]; then - echo "Error: Twine upload failed with exit code $TWINE_EXIT_CODE." - echo "Twine output:" - echo "$TWINE_OUTPUT" - exit $TWINE_EXIT_CODE -fi - -echo "Upload successful." \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/scripts/extract_pypi_version.sh b/examples/experimental/dagster-airlift/scripts/extract_pypi_version.sh deleted file mode 100755 index 051799c2b3e06..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/extract_pypi_version.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Define the path to the setup.py file -SETUP_PY_FILE="setup.py" - -# Check if setup.py exists -if [ ! -f "$SETUP_PY_FILE" ]; then - echo "Error: $SETUP_PY_FILE not found!" >&2 - exit 1 -fi - -# Extract the current version from 'version="x.y.z",' -CURRENT_VERSION=$(sed -n 's/.*version="\([0-9]*\.[0-9]*\.[0-9]*\)",.*/\1/p' "$SETUP_PY_FILE") -if [ -z "$CURRENT_VERSION" ]; then - echo "Error: Version string not found in $SETUP_PY_FILE!" >&2 - exit 1 -fi - -# Output the current version -echo "$CURRENT_VERSION" \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/scripts/find_airlift_commits.sh b/examples/experimental/dagster-airlift/scripts/find_airlift_commits.sh deleted file mode 100755 index 287f1af75837f..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/find_airlift_commits.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Function to decrement version number -decrement_version() { - echo "$1" | awk -F. '{ - if ($3 > 0) $3--; - else if ($2 > 0) {$2--; $3=99;} - else if ($1 > 0) {$1--; $2=99; $3=99;} - print $1"."$2"."$3 - }' -} - -# Get the current version -current_version=$(./scripts/extract_pypi_version.sh) - -# Decrement the version -previous_version=$(decrement_version "$current_version") - -# Get the upper hash -upper_hash=$(git log --format="%h" --grep="\[dagster-airlift\] $current_version" | head -n 1) - -# Get the lower hash -lower_hash=$(git log --format="%h" --grep="\[dagster-airlift\] $previous_version" | head -n 1) - -# If either hash is empty, exit with an error -if [ -z "$upper_hash" ] || [ -z "$lower_hash" ]; then - echo "Error: Could not find one or both version commits." - exit 1 -fi - -# Find commits, format them, print to stdout, and copy to clipboard -git log --format="- %s" ${lower_hash}..${upper_hash}^ | - grep -E "\[dagster-airlift\]" | - sed -E 's/\[(dagster-airlift)\]//g' | - sed -E 's/ \(#[0-9]+\)//g' | - sed -E 's/^- ([a-f0-9]+) /- \1 /' | - tee >(pbcopy) - -# Inform the user that the list has been copied to the clipboard -echo -e "\nCommit list has been copied to the clipboard." \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/scripts/increment_pypi_version.sh b/examples/experimental/dagster-airlift/scripts/increment_pypi_version.sh deleted file mode 100755 index bafe2998ae0da..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/increment_pypi_version.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Get the current version using the extract_pypi_version.sh script -CURRENT_VERSION=$(./scripts/extract_pypi_version.sh) -if [ $? -ne 0 ]; then - echo "Error getting current version" - exit 1 -fi - -echo "Current version: $CURRENT_VERSION" - -# Extract x, y, z components of the version -IFS='.' read -r X Y Z <<< "$CURRENT_VERSION" - -# Increment z -NEW_Z=$((Z + 1)) -NEW_VERSION="$X.$Y.$NEW_Z" -echo "New version: $NEW_VERSION" - -# Define the path to the setup.py file -SETUP_PY_FILE="setup.py" - -# Create a backup of the original file -cp "$SETUP_PY_FILE" "${SETUP_PY_FILE}.bak" - -# Replace the old version with the new version in setup.py -echo "Updating version in $SETUP_PY_FILE..." -sed -i "" "s|version=\"$CURRENT_VERSION\"|version=\"$NEW_VERSION\"|" "$SETUP_PY_FILE" - -# Run the specified commands -echo "Running commands..." -if ! ./scripts/build_and_publish.sh; then - echo "Commands failed. Reverting to original version..." - mv "${SETUP_PY_FILE}.bak" "$SETUP_PY_FILE" - exit 1 -fi - -# Clean up the backup if successful -rm "${SETUP_PY_FILE}.bak" -echo "Commands completed successfully. Version updated to $NEW_VERSION." \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/scripts/mirror_federation_tutorial.sh b/examples/experimental/dagster-airlift/scripts/mirror_federation_tutorial.sh deleted file mode 100755 index 1d537820a5a98..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/mirror_federation_tutorial.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# There's a lot of shared work here with the `update_mirror` step of the OSS release process. -# Code here can be consolidated. - -# Exit immediately if a command exits with a non-zero status -set -e - -# Create a new temporary directory -temp_dir=$(mktemp -d) -echo "Created temporary directory: $temp_dir" - -# Change to the temporary directory -cd "$temp_dir" - -# Initialize git, add remote, fetch, and reset -echo "Initializing git repository..." -git init -echo "Adding remote..." -git remote add origin git@github.com:dagster-io/airlift-federation-tutorial.git -echo "Fetching and resetting..." -git fetch origin -git reset --soft origin/main - -# Go back to the original directory -cd - - -echo "Copying files to temporary directory..." -# Copy files from examples/tutorial_examples to the temporary directory -rsync -av \ - --exclude='.git' \ - --exclude='*.tox' \ - --exclude='.airflow_home' \ - --exclude='.dagster_home' \ - --exclude='*.egg-info' \ - --exclude='airlift_federation_tutorial_tests' \ - --exclude='conftest.py' \ - ../../airlift-federation-tutorial/ "$temp_dir/" - -# Make the version extraction script executable -chmod +x "scripts/extract_pypi_version.sh" -# Get the version number -version=$(./scripts/extract_pypi_version.sh) - -# Change to the temporary directory again -cd "$temp_dir" - -# Add all files to git -git add . - - -echo "Committing with version $version..." -# Commit message is the version number -git commit -m "$version" --allow-empty - -echo "Pushing to origin..." -# Push to origin -git push origin HEAD --force - -echo "Pushed successfully. Cleaning up..." -# Clean up: remove the temporary directory -cd .. -rm -rf "$temp_dir" - -echo "Script completed successfully" \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/scripts/mirror_tutorial.sh b/examples/experimental/dagster-airlift/scripts/mirror_tutorial.sh deleted file mode 100755 index 2b4ceb30fa234..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/mirror_tutorial.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# There's a lot of shared work here with the `update_mirror` step of the OSS release process. -# Code here can be consolidated. - -# Exit immediately if a command exits with a non-zero status -set -e - -# Create a new temporary directory -temp_dir=$(mktemp -d) -echo "Created temporary directory: $temp_dir" - -# Change to the temporary directory -cd "$temp_dir" - -# Initialize git, add remote, fetch, and reset -echo "Initializing git repository..." -git init -echo "Adding remote..." -git remote add origin git@github.com:dagster-io/airlift-tutorial.git -echo "Fetching and resetting..." -git fetch origin -git reset --soft origin/main - -# Go back to the original directory -cd - - -echo "Copying files to temporary directory..." -# Copy files from examples/tutorial_examples to the temporary directory -rsync -av \ - --exclude='.git' \ - --exclude='.airflow_home' \ - --exclude='.dagster_home' \ - --exclude='*.egg-info' \ - --exclude='tutorial_example_tests' \ - --exclude='conftest.py' \ - ../../airlift-migration-tutorial/ "$temp_dir/" - -# Make the version extraction script executable -chmod +x "scripts/extract_pypi_version.sh" -# Get the version number -version=$(./scripts/extract_pypi_version.sh) - -# Change to the temporary directory again -cd "$temp_dir" - -# Add all files to git -git add . - - -echo "Committing with version $version..." -# Commit message is the version number -git commit -m "$version" --allow-empty - -echo "Pushing to origin..." -# Push to origin -git push origin HEAD --force - -echo "Pushed successfully. Cleaning up..." -# Clean up: remove the temporary directory -cd .. -rm -rf "$temp_dir" - -echo "Script completed successfully" \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/scripts/update_readme_snippets.py b/examples/experimental/dagster-airlift/scripts/update_readme_snippets.py deleted file mode 100644 index 9102f6d23c1f7..0000000000000 --- a/examples/experimental/dagster-airlift/scripts/update_readme_snippets.py +++ /dev/null @@ -1,64 +0,0 @@ -import re -import sys -from pathlib import Path -from typing import Optional - -from structlog import get_logger - -logger = get_logger("update_readme_snippets") - -MAIN = 'if __name__ == "__main__":' - - -def _get_regex_match_snippet(file_name: Optional[str] = None) -> re.Pattern[str]: - file_name_capture = "(.+)" if not file_name else re.escape(file_name) - return re.compile( - r"```(python|yaml)\n# " + file_name_capture + "\n(?:[\\S\n\t\v ]*?)\n```", re.MULTILINE - ) - - -def update_readme_snippets(readme_filepath_raw: str): - """Given a path to a README file and a list of paths to snippet files, - update any python code blocks in the README file that match the snippet file names - with the contents of the snippet files. - - e.g. - - ```python - # my_snippet.py - - ... - ``` - - Will be replaced with the contents of `my_snippet.py`, if a file with that name is passed. - """ - readme_file = Path(readme_filepath_raw) - base_dir = readme_file.parent - - readme_contents = readme_file.read_text() - - matches = re.findall(_get_regex_match_snippet(), readme_contents) - for file_type, file_name in matches: - # Attempt to find the snippet file in the base dir - snippet_file: Path = base_dir / file_name - if not snippet_file.exists(): - raise ValueError(f"Could not find snippet file {snippet_file}.") - snippet_contents = snippet_file.read_text() - # Remove the MAIN block if it exists - if MAIN in snippet_contents: - snippet_contents = snippet_contents[: snippet_contents.index(MAIN)] - # Fix newline escaping - snippet_contents = snippet_contents.replace("\\n", "\\\\n") - # Strip whitespace at the beginning and end of the snippet - snippet_contents = snippet_contents.strip() - readme_contents = re.sub( - _get_regex_match_snippet(file_name), - f"```{file_type}\n# {file_name}\n{snippet_contents}\n```", - readme_contents, - ) - - readme_file.write_text(readme_contents) - - -if __name__ == "__main__": - update_readme_snippets(*sys.argv[1:]) diff --git a/examples/experimental/dagster-airlift/setup.cfg b/examples/experimental/dagster-airlift/setup.cfg deleted file mode 100644 index f1482c4bf555c..0000000000000 --- a/examples/experimental/dagster-airlift/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -name = dagster_airlift diff --git a/examples/starlift-demo/.vscode/settings.json b/examples/starlift-demo/.vscode/settings.json index f9debe86b67d4..d2551d6d157ef 100644 --- a/examples/starlift-demo/.vscode/settings.json +++ b/examples/starlift-demo/.vscode/settings.json @@ -3,7 +3,7 @@ "python.analysis.indexing": true, "python.analysis.extraPaths": [ "../../../../../python_modules/dagster", - "../../../../../examples/experimental/dagster-airlift", + "../../../../../python_modules/libraries/dagster-airlift", "../../../../../examples/experimental/dagster-blueprints", "../../../../../python_modules/libraries/dagster-dbt" ], diff --git a/examples/starlift-demo/Makefile b/examples/starlift-demo/Makefile index 0531040ecf1a7..11bd23828f9e2 100644 --- a/examples/starlift-demo/Makefile +++ b/examples/starlift-demo/Makefile @@ -12,7 +12,7 @@ export FEDERATED_AIRFLOW_HOME_2 := $(MAKEFILE_DIR)/.federated_airflow_home_2 export DBT_PROJECT_DIR := $(MAKEFILE_DIR)/dbt_example/shared/dbt export DBT_PROFILES_DIR := $(MAKEFILE_DIR)/dbt_example/shared/dbt export DAGSTER_URL := http://localhost:3333 -export AIRLIFT_MODULE_DIR := $(MAKEFILE_DIR)/../experimental/dagster-airlift +export AIRLIFT_MODULE_DIR := $(MAKEFILE_DIR)/../../python_modules/libraries/dagster-airlift export DLIFT_MODULE_DIR := $(MAKEFILE_DIR)/../experimental/dagster-dlift help: diff --git a/examples/starlift-demo/tox.ini b/examples/starlift-demo/tox.ini index c6e0589e343bb..a588e1142fc46 100644 --- a/examples/starlift-demo/tox.ini +++ b/examples/starlift-demo/tox.ini @@ -16,7 +16,7 @@ deps = -e ../../python_modules/dagster-pipes -e ../../python_modules/dagster-graphql -e ../../python_modules/libraries/dagster-dbt - -e ../experimental/dagster-airlift[core,dbt,test,in-airflow] + -e ../../python_modules/libraries/dagster-airlift[core,dbt,test,in-airflow] -e ../experimental/dagster-dlift -e . pandas diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index cccf8112a8a3a..5506daea5833c 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -104,7 +104,7 @@ daff==1.3.46 -e python_modules/dagster -e python_modules/libraries/dagster-airbyte -e python_modules/libraries/dagster-airflow --e examples/experimental/dagster-airlift +-e python_modules/libraries/dagster-airlift -e python_modules/libraries/dagster-aws -e python_modules/libraries/dagster-azure -e examples/experimental/dagster-blueprints @@ -315,7 +315,7 @@ jupyterlab==4.2.5 jupyterlab-pygments==0.3.0 jupyterlab-server==2.27.3 jupyterlab-widgets==3.0.13 --e examples/experimental/dagster-airlift/kitchen-sink +-e python_modules/libraries/dagster-airlift/kitchen-sink kiwisolver==1.4.7 kombu==5.4.2 kopf==1.37.2 @@ -420,7 +420,7 @@ pathable==0.4.3 pathspec==0.12.1 pathvalidate==3.2.1 pendulum==2.1.2 --e examples/experimental/dagster-airlift/perf-harness +-e python_modules/libraries/dagster-airlift/perf-harness pexpect==4.9.0 pillow==11.0.0 pip==24.3.1 diff --git a/pyright/master/requirements.txt b/pyright/master/requirements.txt index 1fdae6679b2c7..5a83b47ab4e58 100644 --- a/pyright/master/requirements.txt +++ b/pyright/master/requirements.txt @@ -133,11 +133,11 @@ types-sqlalchemy==1.4.53.34 -e examples/with_pyspark -e examples/with_wandb -e examples/experimental/dagster-blueprints --e examples/experimental/dagster-airlift[mwaa,dbt,test] # (includes airflow dependencies) +-e python_modules/libraries/dagster-airlift[mwaa,dbt,test] # (includes airflow dependencies) -e examples/experimental/dagster-dlift -e examples/starlift-demo --e examples/experimental/dagster-airlift/kitchen-sink +-e python_modules/libraries/dagster-airlift/kitchen-sink -e examples/experimental/dagster-dlift/kitchen-sink --e examples/experimental/dagster-airlift/perf-harness +-e python_modules/libraries/dagster-airlift/perf-harness -e examples/airlift-migration-tutorial -e examples/use_case_repository[dev] diff --git a/python_modules/automation/tox.ini b/python_modules/automation/tox.ini index d105b899f0f80..35d009a279a93 100644 --- a/python_modules/automation/tox.ini +++ b/python_modules/automation/tox.ini @@ -67,6 +67,7 @@ deps = -e ../libraries/dagster-pandera -e ../libraries/dagster-polars -e ../libraries/dagster-openai + -e ../libraries/dagster-airlift -e . allowlist_externals = /bin/bash diff --git a/python_modules/dagster/dagster/_generate/download.py b/python_modules/dagster/dagster/_generate/download.py index 2e55762c15056..05abf10322e33 100644 --- a/python_modules/dagster/dagster/_generate/download.py +++ b/python_modules/dagster/dagster/_generate/download.py @@ -16,9 +16,7 @@ "experimental", "use_case_repository", "airlift-mwaa-example", - "airlift-migration-tutorial", "starlift-demo", - "airlift-federation-tutorial", "pyproject.toml", "README.md", "temp_pins.txt", @@ -54,6 +52,8 @@ "with_pyspark", "with_pyspark_emr", "with_wandb", + "airlift-federation-tutorial", + "airlift-migration-tutorial", ] diff --git a/examples/experimental/dagster-airlift/.coveragerc b/python_modules/libraries/dagster-airlift/.coveragerc similarity index 100% rename from examples/experimental/dagster-airlift/.coveragerc rename to python_modules/libraries/dagster-airlift/.coveragerc diff --git a/examples/experimental/dagster-airlift/LICENSE b/python_modules/libraries/dagster-airlift/LICENSE similarity index 100% rename from examples/experimental/dagster-airlift/LICENSE rename to python_modules/libraries/dagster-airlift/LICENSE diff --git a/python_modules/libraries/dagster-airlift/MANIFEST.in b/python_modules/libraries/dagster-airlift/MANIFEST.in new file mode 100644 index 0000000000000..ea3db38ddbd9a --- /dev/null +++ b/python_modules/libraries/dagster-airlift/MANIFEST.in @@ -0,0 +1,9 @@ +include README.md +include LICENSE +include dagster_airlift/py.typed +include Makefile +include CHANGES.md +exclude conftest.py +prune kitchen-sink +prune perf-harness +prune scripts \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/README.md b/python_modules/libraries/dagster-airlift/README.md similarity index 100% rename from examples/experimental/dagster-airlift/README.md rename to python_modules/libraries/dagster-airlift/README.md diff --git a/examples/experimental/dagster-airlift/conftest.py b/python_modules/libraries/dagster-airlift/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/conftest.py rename to python_modules/libraries/dagster-airlift/conftest.py diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/__init__.py new file mode 100644 index 0000000000000..fcec76e9b904c --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/__init__.py @@ -0,0 +1,5 @@ +from dagster._core.libraries import DagsterLibraryRegistry + +from dagster_airlift.version import __version__ + +DagsterLibraryRegistry.register("dagster-azure", __version__) diff --git a/examples/experimental/dagster-airlift/dagster_airlift/cli.py b/python_modules/libraries/dagster-airlift/dagster_airlift/cli.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/cli.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/cli.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/constants.py b/python_modules/libraries/dagster-airlift/dagster_airlift/constants.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/constants.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/constants.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/__init__.py similarity index 61% rename from examples/experimental/dagster-airlift/dagster_airlift/core/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/__init__.py index 8d6baccc3a9a8..0aeba12b2b360 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift/core/__init__.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/core/__init__.py @@ -1,26 +1,27 @@ -from dagster_airlift.core.serialization.serialized_data import DagInfo as DagInfo - -from .airflow_defs_data import AirflowDefinitionsData as AirflowDefinitionsData -from .basic_auth import ( +from dagster_airlift.core.airflow_defs_data import AirflowDefinitionsData as AirflowDefinitionsData +from dagster_airlift.core.basic_auth import ( AirflowAuthBackend as AirflowAuthBackend, AirflowBasicAuthBackend as AirflowBasicAuthBackend, ) -from .load_defs import ( +from dagster_airlift.core.load_defs import ( AirflowInstance as AirflowInstance, DagSelectorFn as DagSelectorFn, build_defs_from_airflow_instance as build_defs_from_airflow_instance, load_airflow_dag_asset_specs as load_airflow_dag_asset_specs, ) -from .multiple_tasks import ( +from dagster_airlift.core.multiple_tasks import ( TaskHandleDict as TaskHandleDict, assets_with_multiple_task_mappings as assets_with_multiple_task_mappings, ) -from .sensor.event_translation import ( +from dagster_airlift.core.sensor.event_translation import ( AssetEvent as AssetEvent, DagsterEventTransformerFn as DagsterEventTransformerFn, ) -from .sensor.sensor_builder import build_airflow_polling_sensor as build_airflow_polling_sensor -from .top_level_dag_def_api import ( +from dagster_airlift.core.sensor.sensor_builder import ( + build_airflow_polling_sensor as build_airflow_polling_sensor, +) +from dagster_airlift.core.serialization.serialized_data import DagInfo as DagInfo +from dagster_airlift.core.top_level_dag_def_api import ( assets_with_dag_mappings as assets_with_dag_mappings, assets_with_task_mappings as assets_with_task_mappings, dag_defs as dag_defs, diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/airflow_defs_data.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/airflow_defs_data.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/airflow_defs_data.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/airflow_defs_data.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/airflow_instance.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/airflow_instance.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/airflow_instance.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/airflow_instance.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/basic_auth.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/basic_auth.py similarity index 94% rename from examples/experimental/dagster-airlift/dagster_airlift/core/basic_auth.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/basic_auth.py index 45b911289b1b9..ecd05551edd66 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift/core/basic_auth.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/core/basic_auth.py @@ -1,6 +1,6 @@ import requests -from .airflow_instance import AirflowAuthBackend +from dagster_airlift.core.airflow_instance import AirflowAuthBackend class AirflowBasicAuthBackend(AirflowAuthBackend): diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/dag_asset.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/dag_asset.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/dag_asset.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/dag_asset.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/load_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/load_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/load_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/load_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/multiple_tasks.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/multiple_tasks.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/multiple_tasks.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/multiple_tasks.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/sensor/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/sensor/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/sensor/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/sensor/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/sensor/event_translation.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/sensor/event_translation.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/sensor/event_translation.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/sensor/event_translation.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/sensor/sensor_builder.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/sensor/sensor_builder.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/sensor/sensor_builder.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/sensor/sensor_builder.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/serialization/compute.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/compute.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/serialization/compute.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/compute.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/serialization/defs_construction.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/defs_construction.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/serialization/defs_construction.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/defs_construction.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/serialization/serialized_data.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/serialized_data.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/serialization/serialized_data.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/serialization/serialized_data.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/top_level_dag_def_api.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/top_level_dag_def_api.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/top_level_dag_def_api.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/top_level_dag_def_api.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/utils.py b/python_modules/libraries/dagster-airlift/dagster_airlift/core/utils.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/utils.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/core/utils.py diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift/dbt/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/dbt/__init__.py new file mode 100644 index 0000000000000..200d8dbb386a7 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/dbt/__init__.py @@ -0,0 +1 @@ +from dagster_airlift.dbt.multi_asset import dbt_defs as dbt_defs diff --git a/examples/experimental/dagster-airlift/dagster_airlift/dbt/multi_asset.py b/python_modules/libraries/dagster-airlift/dagster_airlift/dbt/multi_asset.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/dbt/multi_asset.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/dbt/multi_asset.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/__init__.py similarity index 98% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/__init__.py index 0415103bcd98f..1a08380b5b81f 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/__init__.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/__init__.py @@ -1,3 +1,4 @@ +# ruff: noqa: TID252 def ensure_airflow_installed() -> None: """Ensures that Airflow is installed.""" try: diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/base_asset_operator.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/base_asset_operator.py similarity index 98% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/base_asset_operator.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/base_asset_operator.py index ee98704ee86e2..3394411402814 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/base_asset_operator.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/base_asset_operator.py @@ -11,9 +11,13 @@ from requests import Response from dagster_airlift.constants import DAG_ID_TAG_KEY, DAG_RUN_ID_TAG_KEY, TASK_ID_TAG_KEY - -from .gql_queries import ASSET_NODES_QUERY, RUNS_QUERY, TRIGGER_ASSETS_MUTATION, VERIFICATION_QUERY -from .partition_utils import ( +from dagster_airlift.in_airflow.gql_queries import ( + ASSET_NODES_QUERY, + RUNS_QUERY, + TRIGGER_ASSETS_MUTATION, + VERIFICATION_QUERY, +) +from dagster_airlift.in_airflow.partition_utils import ( PARTITION_NAME_TAG, PartitioningInformation, translate_logical_date_to_partition_key, diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/dag_proxy_operator.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/dag_proxy_operator.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/dag_proxy_operator.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/dag_proxy_operator.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/gql_queries.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/gql_queries.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/gql_queries.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/gql_queries.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/materialize_assets_operator.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/materialize_assets_operator.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/materialize_assets_operator.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/materialize_assets_operator.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/partition_utils.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/partition_utils.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/partition_utils.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/partition_utils.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/proxied_state.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/proxied_state.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/proxied_state.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/proxied_state.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/proxying_fn.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/proxying_fn.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/proxying_fn.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/proxying_fn.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/scaffolding.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/scaffolding.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/scaffolding.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/scaffolding.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/in_airflow/task_proxy_operator.py b/python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/task_proxy_operator.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/in_airflow/task_proxy_operator.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/in_airflow/task_proxy_operator.py diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift/k8s/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/k8s/__init__.py new file mode 100644 index 0000000000000..bcda90d922173 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/k8s/__init__.py @@ -0,0 +1 @@ +from dagster_airlift.k8s.asset import k8s_pod_defs as k8s_pod_defs diff --git a/examples/experimental/dagster-airlift/dagster_airlift/k8s/asset.py b/python_modules/libraries/dagster-airlift/dagster_airlift/k8s/asset.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/k8s/asset.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/k8s/asset.py diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift/mwaa/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/mwaa/__init__.py new file mode 100644 index 0000000000000..4e35f972c29c2 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/mwaa/__init__.py @@ -0,0 +1 @@ +from dagster_airlift.mwaa.auth import MwaaSessionAuthBackend as MwaaSessionAuthBackend diff --git a/examples/experimental/dagster-airlift/dagster_airlift/mwaa/auth.py b/python_modules/libraries/dagster-airlift/dagster_airlift/mwaa/auth.py similarity index 97% rename from examples/experimental/dagster-airlift/dagster_airlift/mwaa/auth.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/mwaa/auth.py index 6fa907134863f..5b80ab8d7276b 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift/mwaa/auth.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/mwaa/auth.py @@ -3,7 +3,7 @@ import boto3 import requests -from ..core.airflow_instance import AirflowAuthBackend +from dagster_airlift.core.airflow_instance import AirflowAuthBackend def get_session_info(mwaa: Any, env_name: str) -> Tuple[str, str]: diff --git a/examples/experimental/dagster-airlift/py.typed b/python_modules/libraries/dagster-airlift/dagster_airlift/py.typed similarity index 100% rename from examples/experimental/dagster-airlift/py.typed rename to python_modules/libraries/dagster-airlift/dagster_airlift/py.typed diff --git a/examples/experimental/dagster-airlift/dagster_airlift/test/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift/test/__init__.py similarity index 95% rename from examples/experimental/dagster-airlift/dagster_airlift/test/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/test/__init__.py index 0aa32f6c75b7b..f6158b4625704 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift/test/__init__.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift/test/__init__.py @@ -1,3 +1,4 @@ +# ruff: noqa: TID252 from .airflow_test_instance import ( AirflowInstanceFake as AirflowInstanceFake, DummyAuthBackend as DummyAuthBackend, diff --git a/examples/experimental/dagster-airlift/dagster_airlift/test/airflow_test_instance.py b/python_modules/libraries/dagster-airlift/dagster_airlift/test/airflow_test_instance.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/test/airflow_test_instance.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/test/airflow_test_instance.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/test/shared_fixtures.py b/python_modules/libraries/dagster-airlift/dagster_airlift/test/shared_fixtures.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/test/shared_fixtures.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/test/shared_fixtures.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/test/test_utils.py b/python_modules/libraries/dagster-airlift/dagster_airlift/test/test_utils.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/test/test_utils.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/test/test_utils.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/version.py b/python_modules/libraries/dagster-airlift/dagster_airlift/version.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/version.py rename to python_modules/libraries/dagster-airlift/dagster_airlift/version.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift/core/serialization/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift/core/serialization/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/conftest.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/conftest.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/conftest.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/dags/simple_dag.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/dags/simple_dag.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/dags/simple_dag.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/dags/simple_dag.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dags/dag.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dags/dag.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dags/dag.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dags/dag.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dagster_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dagster_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dagster_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/materialize_assets_operator_test_project/dagster_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dags/migrated_dag.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dags/migrated_dag.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dags/migrated_dag.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dags/migrated_dag.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dagster_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dagster_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dagster_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/operator_test_project/dagster_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/dags.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/dags.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/dags.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/dags.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_airflow_instance.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_airflow_instance.py similarity index 97% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_airflow_instance.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_airflow_instance.py index dcb9a191f6503..178e16879a417 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_airflow_instance.py +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_airflow_instance.py @@ -4,7 +4,7 @@ from dagster._core.errors import DagsterError from dagster_airlift.core import AirflowBasicAuthBackend, AirflowInstance -from .conftest import assert_link_exists +from dagster_airlift_tests.integration_tests.conftest import assert_link_exists def test_airflow_instance(airflow_instance: None) -> None: diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_materialize_assets_operator.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_materialize_assets_operator.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_materialize_assets_operator.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_materialize_assets_operator.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_proxy_task_operator.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_proxy_task_operator.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_proxy_task_operator.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_proxy_task_operator.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_scaffolding.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_scaffolding.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/test_scaffolding.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/integration_tests/test_scaffolding.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/test_version.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/test_version.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/test_version.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/test_version.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/integration_tests/scaffold_test_airflow_home/airflow_dags/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/conftest.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/conftest.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/conftest.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/proxied_state_for_sqlite_test/dag.yaml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/proxied_state_for_sqlite_test/dag.yaml similarity index 52% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/proxied_state_for_sqlite_test/dag.yaml rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/proxied_state_for_sqlite_test/dag.yaml index 07314a1b25adc..5038a1e1dadc9 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/proxied_state_for_sqlite_test/dag.yaml +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/proxied_state_for_sqlite_test/dag.yaml @@ -1,3 +1,3 @@ tasks: - id: task - proxied: True \ No newline at end of file + proxied: True diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_airflow_asset_mapping.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_airflow_asset_mapping.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_airflow_asset_mapping.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_airflow_asset_mapping.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_asset_keys.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_asset_keys.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_asset_keys.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_asset_keys.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_dag_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_load_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_load_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_load_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_load_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_sensor.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_sensor.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_sensor.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/test_sensor.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/core_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/dbt_project.yml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/dbt_project.yml similarity index 63% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/dbt_project.yml rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/dbt_project.yml index 8bfaa9c67b503..0ce8568cab38c 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/dbt_project.yml +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/dbt_project.yml @@ -1,9 +1,9 @@ -name: 'jaffle_shop' +name: "jaffle_shop" config-version: 2 -version: '0.1' +version: "0.1" -profile: 'jaffle_shop' +profile: "jaffle_shop" model-paths: ["models"] seed-paths: ["seeds"] @@ -13,15 +13,15 @@ macro-paths: ["macros"] target-path: "target" clean-targets: - - "target" - - "dbt_modules" - - "logs" + - "target" + - "dbt_modules" + - "logs" require-dbt-version: [">=1.0.0", "<2.0.0"] -seeds: +seeds: +docs: - node_color: '#cd7f32' + node_color: "#cd7f32" models: jaffle_shop: @@ -29,6 +29,6 @@ models: staging: materialized: view +docs: - node_color: 'silver' + node_color: "silver" +docs: - node_color: 'gold' + node_color: "gold" diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/customers.sql b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/customers.sql similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/customers.sql rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/customers.sql diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/docs.md b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/docs.md similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/docs.md rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/docs.md diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/orders.sql b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/orders.sql similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/orders.sql rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/orders.sql diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/overview.md b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/overview.md similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/overview.md rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/overview.md diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/schema.yml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/schema.yml similarity index 94% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/schema.yml rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/schema.yml index b3d10d05b3ab7..05c5be6930f7f 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/schema.yml +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/schema.yml @@ -54,7 +54,8 @@ models: description: '{{ doc("orders_status") }}' tests: - accepted_values: - values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + values: + ["placed", "shipped", "completed", "return_pending", "returned"] - name: amount description: Total amount (AUD) of the order @@ -79,4 +80,4 @@ models: - name: gift_card_amount description: Amount of the order (AUD) paid for by gift card tests: - - not_null \ No newline at end of file + - not_null diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/schema.yml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/schema.yml similarity index 73% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/schema.yml rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/schema.yml index c207e4cf52950..258345211a9a2 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/schema.yml +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/schema.yml @@ -17,7 +17,8 @@ models: - name: status tests: - accepted_values: - values: ['placed', 'shipped', 'completed', 'return_pending', 'returned'] + values: + ["placed", "shipped", "completed", "return_pending", "returned"] - name: stg_payments columns: @@ -28,4 +29,4 @@ models: - name: payment_method tests: - accepted_values: - values: ['credit_card', 'coupon', 'bank_transfer', 'gift_card'] + values: ["credit_card", "coupon", "bank_transfer", "gift_card"] diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_customers.sql b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_customers.sql similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_customers.sql rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_customers.sql diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_orders.sql b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_orders.sql similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_orders.sql rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_orders.sql diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_payments.sql b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_payments.sql similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_payments.sql rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/models/staging/stg_payments.sql diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/profiles.yml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/profiles.yml similarity index 99% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/profiles.yml rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/profiles.yml index e9fa70297495c..124255a7fb865 100644 --- a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/profiles.yml +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/profiles.yml @@ -1,5 +1,4 @@ jaffle_shop: - target: dev outputs: dev: diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/.gitkeep b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/.gitkeep similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/.gitkeep rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/.gitkeep diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_customers.csv b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_customers.csv similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_customers.csv rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_customers.csv diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_orders.csv b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_orders.csv similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_orders.csv rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_orders.csv diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_payments.csv b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_payments.csv similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_payments.csv rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/dbt_project/seeds/raw_payments.csv diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/test_dbt_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/test_dbt_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/test_dbt_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/test_dbt_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/dbt_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/conftest.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/conftest.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/conftest.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_parse_metadata.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_parse_metadata.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_parse_metadata.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_parse_metadata.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_proxying_to_dagster.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_proxying_to_dagster.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_proxying_to_dagster.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/test_proxying_to_dagster.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/in_airflow_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/test_k8s_pod_defs.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/test_k8s_pod_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/test_k8s_pod_defs.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/test_k8s_pod_defs.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/__init__.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/k8s_tests/__init__.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/test_mwaa_auth.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/test_mwaa_auth.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/test_mwaa_auth.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/test_mwaa_auth.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/empty_file/empty.yaml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/empty_file/empty.yaml similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/empty_file/empty.yaml rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/empty_file/empty.yaml diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/extra_key/extra_key.yaml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/extra_key/extra_key.yaml new file mode 100644 index 0000000000000..59d6bfd0f3ff3 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/extra_key/extra_key.yaml @@ -0,0 +1,4 @@ +tasks: + - id: some_task + proxied: True + extra_key: extra_value diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/nonsense/nonsense_value.yaml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/nonsense/nonsense_value.yaml new file mode 100644 index 0000000000000..fa645101e5222 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/nonsense/nonsense_value.yaml @@ -0,0 +1,3 @@ +tasks: + - id: some_task + proxied: elephant diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/first.yaml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/first.yaml new file mode 100644 index 0000000000000..9b1181bc1140b --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/first.yaml @@ -0,0 +1,7 @@ +tasks: + - id: first_task + proxied: True + - id: second_task + proxied: False + - id: third_task + proxied: True diff --git a/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/second.yaml b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/second.yaml new file mode 100644 index 0000000000000..93f5142eddd5c --- /dev/null +++ b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/proxied_state_yamls/valid/second.yaml @@ -0,0 +1,5 @@ +tasks: + - id: some_task + proxied: True + - id: other_task + proxied: False diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/test_proxied_state.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/test_proxied_state.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/test_proxied_state.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/test_proxied_state.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/test_test_utils.py b/python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/test_test_utils.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/test_test_utils.py rename to python_modules/libraries/dagster-airlift/dagster_airlift_tests/unit_tests/test_test_utils.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/.gitignore b/python_modules/libraries/dagster-airlift/kitchen-sink/.gitignore similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/.gitignore rename to python_modules/libraries/dagster-airlift/kitchen-sink/.gitignore diff --git a/examples/experimental/dagster-airlift/kitchen-sink/Makefile b/python_modules/libraries/dagster-airlift/kitchen-sink/Makefile similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/Makefile rename to python_modules/libraries/dagster-airlift/kitchen-sink/Makefile diff --git a/examples/experimental/dagster-airlift/kitchen-sink/README.md b/python_modules/libraries/dagster-airlift/kitchen-sink/README.md similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/README.md rename to python_modules/libraries/dagster-airlift/kitchen-sink/README.md diff --git a/examples/experimental/dagster-airlift/kitchen-sink/conftest.py b/python_modules/libraries/dagster-airlift/kitchen-sink/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/conftest.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/conftest.py diff --git a/examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/__init__.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/dagster_airlift_tests/unit_tests/mwaa_tests/__init__.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/__init__.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/custom_callback.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/custom_callback.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/custom_callback.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/custom_callback.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_custom_callback.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_custom_callback.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_custom_callback.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_custom_callback.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_proxied.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_proxied.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_proxied.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/dag_level_proxied.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/interval_dag.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/interval_dag.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/interval_dag.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/interval_dag.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/migrated_partitioned.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/migrated_partitioned.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/migrated_partitioned.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/migrated_partitioned.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_job_assets_dag.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_job_assets_dag.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_job_assets_dag.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_job_assets_dag.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_location_dags.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_location_dags.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_location_dags.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/multi_location_dags.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/print_dag.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/print_dag.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/print_dag.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/print_dag.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/affected_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/affected_dag.yaml similarity index 80% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/affected_dag.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/affected_dag.yaml index ca6a4a78ada18..a803c2be5f771 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/affected_dag.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/affected_dag.yaml @@ -2,4 +2,4 @@ tasks: - id: print_task proxied: True - id: downstream_print_task - proxied: True \ No newline at end of file + proxied: True diff --git a/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml new file mode 100644 index 0000000000000..7abd5af9a3138 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_first_code_location.yaml @@ -0,0 +1,3 @@ +tasks: + - id: task + proxied: False diff --git a/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml new file mode 100644 index 0000000000000..7abd5af9a3138 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/dag_second_code_location.yaml @@ -0,0 +1,3 @@ +tasks: + - id: task + proxied: False diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/daily_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/daily_dag.yaml similarity index 62% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/daily_dag.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/daily_dag.yaml index 702078538fd46..3b62219ac39fa 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/daily_dag.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/daily_dag.yaml @@ -1,3 +1,3 @@ tasks: - id: asset_one_daily - proxied: False \ No newline at end of file + proxied: False diff --git a/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/migrated_daily_interval_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/migrated_daily_interval_dag.yaml new file mode 100644 index 0000000000000..137c1a568888d --- /dev/null +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/migrated_daily_interval_dag.yaml @@ -0,0 +1,3 @@ +tasks: + - id: my_task + proxied: True diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/multi_job_assets_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/multi_job_assets_dag.yaml similarity index 59% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/multi_job_assets_dag.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/multi_job_assets_dag.yaml index 68898e0c9072c..a8aca80887ff0 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/multi_job_assets_dag.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/multi_job_assets_dag.yaml @@ -1,3 +1,3 @@ tasks: - id: print_task - proxied: True \ No newline at end of file + proxied: True diff --git a/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag.yaml new file mode 100644 index 0000000000000..acb65a33ca1a5 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag.yaml @@ -0,0 +1 @@ +proxied: True diff --git a/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag_custom_callback.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag_custom_callback.yaml new file mode 100644 index 0000000000000..acb65a33ca1a5 --- /dev/null +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/overridden_dag_custom_callback.yaml @@ -0,0 +1 @@ +proxied: True diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/print_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/print_dag.yaml similarity index 80% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/print_dag.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/print_dag.yaml index ca6a4a78ada18..a803c2be5f771 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/print_dag.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/print_dag.yaml @@ -2,4 +2,4 @@ tasks: - id: print_task proxied: True - id: downstream_print_task - proxied: True \ No newline at end of file + proxied: True diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/unaffected_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/unaffected_dag.yaml similarity index 80% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/unaffected_dag.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/unaffected_dag.yaml index ca6a4a78ada18..a803c2be5f771 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/unaffected_dag.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/unaffected_dag.yaml @@ -2,4 +2,4 @@ tasks: - id: print_task proxied: True - id: downstream_print_task - proxied: True \ No newline at end of file + proxied: True diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/weekly_dag.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/weekly_dag.yaml similarity index 62% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/weekly_dag.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/weekly_dag.yaml index b690dd97f3bb4..a317921aaf1f3 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/weekly_dag.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/proxied_state/weekly_dag.yaml @@ -1,3 +1,3 @@ tasks: - id: asset_one_weekly - proxied: False \ No newline at end of file + proxied: False diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/simple_unproxied.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/simple_unproxied.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/simple_unproxied.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_dags/simple_unproxied.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_instance.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_instance.py similarity index 77% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_instance.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_instance.py index aa9e8fd69ec7c..7fef12ff167e0 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/airflow_instance.py +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/airflow_instance.py @@ -1,6 +1,6 @@ from dagster_airlift.core import AirflowBasicAuthBackend, AirflowInstance -from .constants import AIRFLOW_BASE_URL, AIRFLOW_INSTANCE_NAME, PASSWORD, USERNAME +from kitchen_sink.constants import AIRFLOW_BASE_URL, AIRFLOW_INSTANCE_NAME, PASSWORD, USERNAME def local_airflow_instance() -> AirflowInstance: diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/constants.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/constants.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/constants.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/constants.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/__init__.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/__init__.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/__init__.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/automapped_defs.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/automapped_defs.py similarity index 92% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/automapped_defs.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/automapped_defs.py index 467f3849118d4..6d3a5aed9bf4e 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/automapped_defs.py +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/automapped_defs.py @@ -2,7 +2,7 @@ from dagster_airlift.core import dag_defs, task_defs from dagster_airlift.core.load_defs import build_full_automapped_dags_from_airflow_instance -from ..airflow_instance import local_airflow_instance +from kitchen_sink.airflow_instance import local_airflow_instance @asset diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/mapped_defs.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/mapped_defs.py similarity index 98% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/mapped_defs.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/mapped_defs.py index 724b0bd204238..f3593bddc4ee8 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/mapped_defs.py +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/mapped_defs.py @@ -14,7 +14,7 @@ ) from dagster_airlift.core.multiple_tasks import targeted_by_multiple_tasks -from ..airflow_instance import local_airflow_instance +from kitchen_sink.airflow_instance import local_airflow_instance def make_print_asset(key: str) -> AssetsDefinition: diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/observation_defs.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/observation_defs.py similarity index 95% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/observation_defs.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/observation_defs.py index 45f5733cc9e29..97ab1db527750 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/observation_defs.py +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/observation_defs.py @@ -15,7 +15,7 @@ task_defs, ) -from ..airflow_instance import local_airflow_instance +from kitchen_sink.airflow_instance import local_airflow_instance def observations_from_materializations( diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/__init__.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_defs/__init__.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/__init__.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/first_dag_defs.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/first_dag_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/first_dag_defs.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/first_dag_defs.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/second_dag_defs.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/second_dag_defs.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/second_dag_defs.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/second_dag_defs.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml similarity index 85% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml index e73a9a5b8b630..f5dd0be843ebc 100644 --- a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml +++ b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/workspace.yaml @@ -4,4 +4,4 @@ load_from: location_name: first_dag_location - python_module: module_name: kitchen_sink.dagster_multi_code_locations.second_dag_defs - location_name: second_dag_location \ No newline at end of file + location_name: second_dag_location diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/__init__.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink/dagster_multi_code_locations/__init__.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/__init__.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/__init__.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/conftest.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/conftest.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/conftest.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_automapped.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_automapped.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_automapped.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_automapped.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_mapped.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_mapped.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_mapped.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_mapped.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_multi_code_location.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_multi_code_location.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_multi_code_location.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_multi_code_location.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_observation.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_observation.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_observation.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/test_e2e_observation.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/__init__.py b/python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/unit_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/integration_tests/__init__.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/kitchen_sink_tests/unit_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/pyproject.toml b/python_modules/libraries/dagster-airlift/kitchen-sink/pyproject.toml similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/pyproject.toml rename to python_modules/libraries/dagster-airlift/kitchen-sink/pyproject.toml diff --git a/examples/experimental/dagster-airlift/kitchen-sink/setup.py b/python_modules/libraries/dagster-airlift/kitchen-sink/setup.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/setup.py rename to python_modules/libraries/dagster-airlift/kitchen-sink/setup.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/tox.ini b/python_modules/libraries/dagster-airlift/kitchen-sink/tox.ini similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/tox.ini rename to python_modules/libraries/dagster-airlift/kitchen-sink/tox.ini diff --git a/examples/experimental/dagster-airlift/perf-harness/.gitignore b/python_modules/libraries/dagster-airlift/perf-harness/.gitignore similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/.gitignore rename to python_modules/libraries/dagster-airlift/perf-harness/.gitignore diff --git a/examples/experimental/dagster-airlift/perf-harness/Makefile b/python_modules/libraries/dagster-airlift/perf-harness/Makefile similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/Makefile rename to python_modules/libraries/dagster-airlift/perf-harness/Makefile diff --git a/examples/experimental/dagster-airlift/perf-harness/README.md b/python_modules/libraries/dagster-airlift/perf-harness/README.md similarity index 68% rename from examples/experimental/dagster-airlift/perf-harness/README.md rename to python_modules/libraries/dagster-airlift/perf-harness/README.md index 5f49021032904..50a08823cef75 100644 --- a/examples/experimental/dagster-airlift/perf-harness/README.md +++ b/python_modules/libraries/dagster-airlift/perf-harness/README.md @@ -7,7 +7,7 @@ This is used to time various airlift components. From the Dagster directory ```bash -cd experimental/dagster-airlift/perf-harness +cd python_modules/libraries/dagster-airlift/perf-harness pip install uv uv pip install -e . ``` @@ -18,7 +18,7 @@ Sanity check run perf-harness 1 1 ``` -Then check `experimental/dagster-airlift/perf-harness/perf_harness/shared`. You should see a file +Then check `python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared`. You should see a file `1_dags_1_tasks_perf_output.txt` with airlift timings. The first argument to `perf-harness` is the number of dags, the second is the number of tasks. diff --git a/examples/experimental/dagster-airlift/perf-harness/conftest.py b/python_modules/libraries/dagster-airlift/perf-harness/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/conftest.py rename to python_modules/libraries/dagster-airlift/perf-harness/conftest.py diff --git a/examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/unit_tests/__init__.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/kitchen-sink/kitchen_sink_tests/unit_tests/__init__.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/__init__.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/__init__.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/__init__.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/__init__.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/dags.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/dags.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/dags.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/dags.py diff --git a/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/migration_state/dag_0.yaml b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/migration_state/dag_0.yaml new file mode 100644 index 0000000000000..b9a0c9fbe28bd --- /dev/null +++ b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/airflow_dags/migration_state/dag_0.yaml @@ -0,0 +1,3 @@ +tasks: + - id: task_0_0 + proxied: false diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/cli.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/cli.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/cli.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/cli.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/__init__.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/airflow_dags/__init__.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/__init__.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/constants.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/constants.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/constants.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/constants.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/migrate.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/migrate.py similarity index 93% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/migrate.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/migrate.py index 5dd6a14c85f47..21b9060591ae5 100644 --- a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/migrate.py +++ b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/migrate.py @@ -11,10 +11,14 @@ task_defs, ) +from perf_harness.dagster_defs.constants import ( + AIRFLOW_BASE_URL, + AIRFLOW_INSTANCE_NAME, + PASSWORD, + USERNAME, +) from perf_harness.shared.constants import get_num_assets, get_num_dags, get_num_tasks -from .constants import AIRFLOW_BASE_URL, AIRFLOW_INSTANCE_NAME, PASSWORD, USERNAME - airflow_instance = AirflowInstance( auth_backend=AirflowBasicAuthBackend( webserver_url=AIRFLOW_BASE_URL, username=USERNAME, password=PASSWORD diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/observe.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/observe.py similarity index 92% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/observe.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/observe.py index 1f00f064310de..ecea5f8599b49 100644 --- a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/observe.py +++ b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/observe.py @@ -10,10 +10,14 @@ task_defs, ) +from perf_harness.dagster_defs.constants import ( + AIRFLOW_BASE_URL, + AIRFLOW_INSTANCE_NAME, + PASSWORD, + USERNAME, +) from perf_harness.shared.constants import get_num_assets, get_num_dags, get_num_tasks -from .constants import AIRFLOW_BASE_URL, AIRFLOW_INSTANCE_NAME, PASSWORD, USERNAME - airflow_instance = AirflowInstance( auth_backend=AirflowBasicAuthBackend( webserver_url=AIRFLOW_BASE_URL, username=USERNAME, password=PASSWORD diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/peer.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/peer.py similarity index 75% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/peer.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/peer.py index 3f5aa43b61583..91fb80d944655 100644 --- a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/peer.py +++ b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/dagster_defs/peer.py @@ -4,7 +4,12 @@ build_defs_from_airflow_instance, ) -from .constants import AIRFLOW_BASE_URL, AIRFLOW_INSTANCE_NAME, PASSWORD, USERNAME +from perf_harness.dagster_defs.constants import ( + AIRFLOW_BASE_URL, + AIRFLOW_INSTANCE_NAME, + PASSWORD, + USERNAME, +) airflow_instance = AirflowInstance( auth_backend=AirflowBasicAuthBackend( diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/shared/constants.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared/constants.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/shared/constants.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared/constants.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/shared/constants.txt b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared/constants.txt similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/shared/constants.txt rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared/constants.txt diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/shared/utils.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared/utils.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/shared/utils.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness/shared/utils.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/__init__.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/__init__.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness/dagster_defs/__init__.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/__init__.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/conftest.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/conftest.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/conftest.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/conftest.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/test_cli.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/test_cli.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/test_cli.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/test_cli.py diff --git a/examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/test_e2e.py b/python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/test_e2e.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/perf_harness_tests/test_e2e.py rename to python_modules/libraries/dagster-airlift/perf-harness/perf_harness_tests/test_e2e.py diff --git a/examples/experimental/dagster-airlift/perf-harness/pyproject.toml b/python_modules/libraries/dagster-airlift/perf-harness/pyproject.toml similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/pyproject.toml rename to python_modules/libraries/dagster-airlift/perf-harness/pyproject.toml diff --git a/examples/experimental/dagster-airlift/perf-harness/scripts/generate_yaml_files.py b/python_modules/libraries/dagster-airlift/perf-harness/scripts/generate_yaml_files.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/scripts/generate_yaml_files.py rename to python_modules/libraries/dagster-airlift/perf-harness/scripts/generate_yaml_files.py diff --git a/examples/experimental/dagster-airlift/perf-harness/setup.py b/python_modules/libraries/dagster-airlift/perf-harness/setup.py similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/setup.py rename to python_modules/libraries/dagster-airlift/perf-harness/setup.py diff --git a/examples/experimental/dagster-airlift/perf-harness/tox.ini b/python_modules/libraries/dagster-airlift/perf-harness/tox.ini similarity index 100% rename from examples/experimental/dagster-airlift/perf-harness/tox.ini rename to python_modules/libraries/dagster-airlift/perf-harness/tox.ini diff --git a/examples/experimental/dagster-airlift/scripts/airflow_setup.sh b/python_modules/libraries/dagster-airlift/scripts/airflow_setup.sh similarity index 100% rename from examples/experimental/dagster-airlift/scripts/airflow_setup.sh rename to python_modules/libraries/dagster-airlift/scripts/airflow_setup.sh diff --git a/examples/experimental/dagster-airlift/scripts/find_and_replace_in_yaml_dir.sh b/python_modules/libraries/dagster-airlift/scripts/find_and_replace_in_yaml_dir.sh similarity index 100% rename from examples/experimental/dagster-airlift/scripts/find_and_replace_in_yaml_dir.sh rename to python_modules/libraries/dagster-airlift/scripts/find_and_replace_in_yaml_dir.sh diff --git a/python_modules/libraries/dagster-airlift/setup.cfg b/python_modules/libraries/dagster-airlift/setup.cfg new file mode 100644 index 0000000000000..f288133d3981c --- /dev/null +++ b/python_modules/libraries/dagster-airlift/setup.cfg @@ -0,0 +1,8 @@ +[metadata] +name = dagster_airlift +[check-manifest] +ignore = + .coveragerc + tox.ini + pytest.ini + dagster_airlift_tests/** \ No newline at end of file diff --git a/examples/experimental/dagster-airlift/setup.py b/python_modules/libraries/dagster-airlift/setup.py similarity index 91% rename from examples/experimental/dagster-airlift/setup.py rename to python_modules/libraries/dagster-airlift/setup.py index a2de0715204ee..c7b24fbad9c2d 100644 --- a/examples/experimental/dagster-airlift/setup.py +++ b/python_modules/libraries/dagster-airlift/setup.py @@ -3,8 +3,6 @@ from setuptools import find_packages, setup -NON_EDITABLE_INSTALL_DAGSTER_PIN = ">=1.8.10" - def get_version() -> str: version: Dict[str, str] = {} @@ -15,7 +13,7 @@ def get_version() -> str: ver = get_version() -pin = "" if ver == "1!0+dev" else NON_EDITABLE_INSTALL_DAGSTER_PIN +pin = "" if ver == "1!0+dev" or "rc" in ver else f"=={ver}" # The [in-airflow] subpackage does not have a setup dependency on Airflow because # Airflow cannot be installed via setup.py reliably. Instead, users need to install @@ -44,7 +42,7 @@ def get_version() -> str: setup( name="dagster-airlift", - version="0.0.28", + version=ver, author="Dagster Labs", author_email="hello@dagsterlabs.com", license="Apache-2.0", @@ -63,7 +61,7 @@ def get_version() -> str: "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - packages=find_packages(exclude=["dagster_airlift_tests*", "examples*"]), + packages=find_packages(exclude=["dagster_airlift_tests*", "kitchen-sink", "perf-harness"]), requires=CLI_REQUIREMENTS, extras_require={ "core": [ @@ -82,14 +80,14 @@ def get_version() -> str: "mwaa": [ "boto3>=1.18.0" ], # confirms that mwaa is available in the environment (can't find exactly which version adds mwaa support, but I can confirm that 1.18.0 and greater have it.) - "dbt": ["dagster-dbt"], - "k8s": ["dagster-k8s"], + "dbt": [f"dagster-dbt{pin}"], + "k8s": [f"dagster-k8s{pin}"], "test": [ "pytest", - "dagster-dbt", + f"dagster-dbt{pin}", "dbt-duckdb", "boto3", - "dagster-webserver", + f"dagster-webserver{pin}", *AIRFLOW_REQUIREMENTS, *CLI_REQUIREMENTS, ], diff --git a/examples/experimental/dagster-airlift/tox.ini b/python_modules/libraries/dagster-airlift/tox.ini similarity index 100% rename from examples/experimental/dagster-airlift/tox.ini rename to python_modules/libraries/dagster-airlift/tox.ini diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index 494534db2687c..eedf161783c34 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -51,7 +51,7 @@ def main( editable_target_paths = [ ".buildkite/dagster-buildkite", "examples/experimental/dagster-blueprints", - "examples/experimental/dagster-airlift[core,in-airflow,mwaa,dbt,test]", + "python_modules/libraries/dagster-airlift[core,in-airflow,mwaa,dbt,test]", "integration_tests/python_modules/dagster-k8s-test-infra", "helm/dagster/schema[test]", "python_modules/automation",