diff --git a/examples/airlift-federation-tutorial/README.md b/examples/airlift-federation-tutorial/README.md index cde4d8dd4312d..17a6cf4d6db87 100644 --- a/examples/airlift-federation-tutorial/README.md +++ b/examples/airlift-federation-tutorial/README.md @@ -1,3 +1,19 @@ ## Airlift Federation Tutorial Code -Work in progress. +This repo is the code that the Airlift federation tutorial is based off of. Follow along on the main docs site [here](https://docs.dagster.io/integrations/airlift). + +This example demonstrates how to use the `dagster-airlift` package to unify multiple Airflow instances into Dagster as a single control plane, and then federate execution between those Airflow instances. + +## Example Structure + +The following explains the structure of the repo. + +```plaintext +airlift_federation_tutorial +├── constants.py: Contains constant values used throughout both Airflow and Dagster +├── dagster_defs: Contains Dagster definitions +│ ├── definitions.py: Empty starter file for following along with the tutorial +│ └── stages: Contains reference implementations for each stage of the migration process. +├── metrics_airflow_dags: Contains the Airflow DAGs for the "downstream" airflow instance +└── warehouse_airflow_dags: Contains the Airflow DAGs for the "upstream" airflow instance +``` diff --git a/examples/experimental/dagster-airlift/Makefile b/examples/experimental/dagster-airlift/Makefile index 14fb1b1795df6..1f570472a0159 100644 --- a/examples/experimental/dagster-airlift/Makefile +++ b/examples/experimental/dagster-airlift/Makefile @@ -10,6 +10,10 @@ mirror_tutorial: @chmod +x scripts/mirror_tutorial.sh @./scripts/mirror_tutorial.sh +mirror_federation_tutorial: + @chmod +x scripts/mirror_federation_tutorial.sh + @./scripts/mirror_federation_tutorial.sh + # Runs the full release process for dagster-airlift. # - Enforces that we're on master # - Bumps the version in setup.py @@ -26,6 +30,8 @@ adhoc_release: @make adhoc_pypi @echo "Mirroring tutorial..." @make mirror_tutorial + @echo "Mirroring federation tutorial..." + @make mirror_federation_tutorial @git add . @git checkout -b airlift-$$(./scripts/extract_pypi_version.sh) @git commit -m "[dagster-airlift] $$(./scripts/extract_pypi_version.sh)" diff --git a/examples/experimental/dagster-airlift/scripts/mirror_federation_tutorial.sh b/examples/experimental/dagster-airlift/scripts/mirror_federation_tutorial.sh new file mode 100755 index 0000000000000..1d537820a5a98 --- /dev/null +++ b/examples/experimental/dagster-airlift/scripts/mirror_federation_tutorial.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# There's a lot of shared work here with the `update_mirror` step of the OSS release process. +# Code here can be consolidated. + +# Exit immediately if a command exits with a non-zero status +set -e + +# Create a new temporary directory +temp_dir=$(mktemp -d) +echo "Created temporary directory: $temp_dir" + +# Change to the temporary directory +cd "$temp_dir" + +# Initialize git, add remote, fetch, and reset +echo "Initializing git repository..." +git init +echo "Adding remote..." +git remote add origin git@github.com:dagster-io/airlift-federation-tutorial.git +echo "Fetching and resetting..." +git fetch origin +git reset --soft origin/main + +# Go back to the original directory +cd - + +echo "Copying files to temporary directory..." +# Copy files from examples/tutorial_examples to the temporary directory +rsync -av \ + --exclude='.git' \ + --exclude='*.tox' \ + --exclude='.airflow_home' \ + --exclude='.dagster_home' \ + --exclude='*.egg-info' \ + --exclude='airlift_federation_tutorial_tests' \ + --exclude='conftest.py' \ + ../../airlift-federation-tutorial/ "$temp_dir/" + +# Make the version extraction script executable +chmod +x "scripts/extract_pypi_version.sh" +# Get the version number +version=$(./scripts/extract_pypi_version.sh) + +# Change to the temporary directory again +cd "$temp_dir" + +# Add all files to git +git add . + + +echo "Committing with version $version..." +# Commit message is the version number +git commit -m "$version" --allow-empty + +echo "Pushing to origin..." +# Push to origin +git push origin HEAD --force + +echo "Pushed successfully. Cleaning up..." +# Clean up: remove the temporary directory +cd .. +rm -rf "$temp_dir" + +echo "Script completed successfully" \ No newline at end of file