From 6fb3c4bd9ae64f400e7efe34cbc1042b28e2b8d4 Mon Sep 17 00:00:00 2001 From: alangenfeld Date: Mon, 8 Apr 2024 15:00:09 -0500 Subject: [PATCH] DbtProject support in serverless template --- github/serverless/dbt/branch_deployments.yml | 41 ++++++++++++++----- github/serverless/dbt/deploy.yml | 42 ++++++++++++++------ 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/github/serverless/dbt/branch_deployments.yml b/github/serverless/dbt/branch_deployments.yml index 624b751d..cd94f2a9 100644 --- a/github/serverless/dbt/branch_deployments.yml +++ b/github/serverless/dbt/branch_deployments.yml @@ -33,25 +33,46 @@ jobs: with: dagster_cloud_file: $DAGSTER_CLOUD_FILE - - name: Checkout for Python Executable Deploy + - name: Checkout if: steps.prerun.outputs.result == 'pex-deploy' uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} path: project-repo - - name: Parse dbt project and package with Dagster project + # Validate dagster_cloud.yaml and the connection to dagster.cloud + - name: Validate configuration + id: ci-validate + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" + env: + DAGSTER_PROJECT_DIR: 'dagster_dbt_scaffold' + + + # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session + - name: Initialize build session + id: ci-init + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 + with: + project_dir: 'dagster_dbt_scaffold' + dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} + # The full deployment name. If this run is for a PR, this value is ignored and a branch + # deployment is used. + deployment: 'prod' + + - name: Prepare DBT project for deployment if: steps.prerun.outputs.result == 'pex-deploy' + # --upgrade-strategy eager picks up newer packages that are required for things to work run: | pip install pip --upgrade - cd ${{ env.DBT_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }} - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0' - pip install pyOpenSSL --upgrade - cd - - rsync -avz --exclude ${{ env.DAGSTER_PROJECT_NAME }} --exclude .git ${{ env.DBT_PROJECT_DIR }}/ ${{ env.DBT_PACKAGE_DATA_DIR }} - dbt deps --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }} - dbt parse --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }} - rm ${{ env.DBT_PACKAGE_DATA_DIR }}/target/partial_parse.msgpack + cd ${{ env.ROOT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }} + pip install . --upgrade --upgrade-strategy eager + dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py + # The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject + # dagster-cloud ci dagster-dbt project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py shell: bash env: DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold' diff --git a/github/serverless/dbt/deploy.yml b/github/serverless/dbt/deploy.yml index fafb0d72..5d706738 100644 --- a/github/serverless/dbt/deploy.yml +++ b/github/serverless/dbt/deploy.yml @@ -35,30 +35,48 @@ jobs: with: dagster_cloud_file: $DAGSTER_CLOUD_FILE - - name: Checkout for Python Executable Deploy + - name: Checkout if: steps.prerun.outputs.result == 'pex-deploy' uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} path: project-repo - - name: Parse dbt project and package with Dagster project + # Validate dagster_cloud.yaml and the connection to dagster.cloud + - name: Validate configuration + id: ci-validate + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" + env: + DAGSTER_PROJECT_DIR: 'dagster_dbt_scaffold' + + # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session + - name: Initialize build session + id: ci-init + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 + with: + project_dir: 'dagster_dbt_scaffold' + dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} + # The full deployment name. If this run is for a PR, this value is ignored and a branch + # deployment is used. + deployment: 'prod' + + - name: Prepare DBT project for deployment if: steps.prerun.outputs.result == 'pex-deploy' + # --upgrade-strategy eager picks up newer packages that are required for things to work run: | pip install pip --upgrade - cd ${{ env.DBT_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }} - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0' - pip install pyOpenSSL --upgrade - cd - - rsync -avz --exclude ${{ env.DAGSTER_PROJECT_NAME }} --exclude .git ${{ env.DBT_PROJECT_DIR }}/ ${{ env.DBT_PACKAGE_DATA_DIR }} - dbt deps --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }} - dbt parse --project-dir ${{ env.DBT_PACKAGE_DATA_DIR }} --profiles-dir ${{ env.DBT_PACKAGE_DATA_DIR }} - rm ${{ env.DBT_PACKAGE_DATA_DIR }}/target/partial_parse.msgpack + cd ${{ env.ROOT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }} + pip install . --upgrade --upgrade-strategy eager + dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py + # The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject + # dagster-cloud ci dagster-dbt project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py shell: bash env: DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold' - DBT_PROJECT_DIR: "$GITHUB_WORKSPACE/project-repo" - DBT_PACKAGE_DATA_DIR: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_PROJECT_NAME/dbt-project" - name: Python Executable Deploy if: steps.prerun.outputs.result == 'pex-deploy'