DbtProject support in serverless template

dagster-io · Apr 10, 2024 · 6fb3c4b · 6fb3c4b
1 parent 7c2f65f
commit 6fb3c4b
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 22 deletions.
diff --git a/github/serverless/dbt/branch_deployments.yml b/github/serverless/dbt/branch_deployments.yml
@@ -33,25 +33,46 @@ jobs:
         with:
           dagster_cloud_file: $DAGSTER_CLOUD_FILE
 
-      - name: Checkout for Python Executable Deploy
+      - name: Checkout
         if: steps.prerun.outputs.result == 'pex-deploy'
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           path: project-repo
 
-      - name: Parse dbt project and package with Dagster project
+      # Validate dagster_cloud.yaml and the connection to dagster.cloud
+      - name: Validate configuration
+        id: ci-validate
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}"
+        env:
+          DAGSTER_PROJECT_DIR: 'dagster_dbt_scaffold'
+
+
+      # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
+      - name: Initialize build session
+        id: ci-init
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          project_dir: 'dagster_dbt_scaffold'
+          dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }}
+          # The full deployment name. If this run is for a PR, this value is ignored and a branch
+          # deployment is used.
+          deployment: 'prod'
+
+      - name: Prepare DBT project for deployment
         if: steps.prerun.outputs.result == 'pex-deploy'
+        # --upgrade-strategy eager picks up newer packages that are required for things to work
         run: |
           pip install pip --upgrade
-          cd ${{ env.DBT_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
-          pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
-          pip install pyOpenSSL --upgrade
-          cd -
-          rsync -avz --exclude ${{ env.DAGSTER_PROJECT_NAME }} --exclude .git ${{ env.DBT_PROJECT_DIR }}/ ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt deps --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt parse --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          rm ${{ env.DBT_PACKAGE_DATA_DIR }}/target/partial_parse.msgpack
+          cd ${{ env.ROOT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
+          pip install . --upgrade --upgrade-strategy eager
+          dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
+          # The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject
+          # dagster-cloud ci dagster-dbt project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
         shell: bash
         env:
           DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'

diff --git a/github/serverless/dbt/deploy.yml b/github/serverless/dbt/deploy.yml
@@ -35,30 +35,48 @@ jobs:
         with:
           dagster_cloud_file: $DAGSTER_CLOUD_FILE
 
-      - name: Checkout for Python Executable Deploy
+      - name: Checkout
         if: steps.prerun.outputs.result == 'pex-deploy'
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
           path: project-repo
 
-      - name: Parse dbt project and package with Dagster project
+      # Validate dagster_cloud.yaml and the connection to dagster.cloud
+      - name: Validate configuration
+        id: ci-validate
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}"
+        env:
+          DAGSTER_PROJECT_DIR: 'dagster_dbt_scaffold'
+
+      # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
+      - name: Initialize build session
+        id: ci-init
+        if: steps.prerun.outputs.result == 'pex-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          project_dir: 'dagster_dbt_scaffold'
+          dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }}
+          # The full deployment name. If this run is for a PR, this value is ignored and a branch
+          # deployment is used.
+          deployment: 'prod'
+
+      - name: Prepare DBT project for deployment
         if: steps.prerun.outputs.result == 'pex-deploy'
+        # --upgrade-strategy eager picks up newer packages that are required for things to work
         run: |
           pip install pip --upgrade
-          cd ${{ env.DBT_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
-          pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
-          pip install pyOpenSSL --upgrade
-          cd -
-          rsync -avz --exclude ${{ env.DAGSTER_PROJECT_NAME }} --exclude .git ${{ env.DBT_PROJECT_DIR }}/ ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt deps --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          dbt parse --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }}
-          rm ${{ env.DBT_PACKAGE_DATA_DIR }}/target/partial_parse.msgpack
+          cd ${{ env.ROOT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
+          pip install . --upgrade --upgrade-strategy eager
+          dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
+          # The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject
+          # dagster-cloud ci dagster-dbt project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
         shell: bash
         env:
           DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'
-          DBT_PROJECT_DIR: "$GITHUB_WORKSPACE/project-repo"
-          DBT_PACKAGE_DATA_DIR: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_PROJECT_NAME/dbt-project"
 
       - name: Python Executable Deploy
         if: steps.prerun.outputs.result == 'pex-deploy'