deploy-dagster-cloud.yml

name: Dagster Cloud Hybrid Deployment
on:
  push: # For full deployment
    branches:
      - "main"
      - "master"
  pull_request:  # For branch deployments
     types: [opened, synchronize, reopened, closed]

concurrency:
  # Cancel in-progress deploys to the same branch
  group: ${{ github.ref }}
  cancel-in-progress: true
env:
  DAGSTER_CLOUD_ORGANIZATION: "hooli"
  DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
  DAGSTER_PROJECT_DIR: "."
  DAGSTER_CLOUD_YAML_PATH: "dagster_cloud.yaml"
  # The IMAGE_REGISTRY should match the registry: in dagster_cloud.yaml
  IMAGE_REGISTRY: "764506304434.dkr.ecr.us-west-2.amazonaws.com/hooli-data-science-prod"
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
  dagster-cloud-deploy:
    runs-on: ubuntu-20.04
    steps:
      - name: Pre-run checks
        id: prerun
        uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1.27

      - name: Checkout
        uses: actions/checkout@v3
        if: steps.prerun.outputs.result != 'skip'
        with:
          ref: ${{ github.head_ref }}

      - name: Validate configuration
        id: ci-validate
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_YAML_PATH }}"

      - name: Initialize build session
        id: ci-init
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1.27
        with:
          project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
          dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }}
          deployment: 'data-eng-prod'

      - name: Generate docker image tag
        id: generate-image-tag
        if: steps.prerun.outputs.result != 'skip'
        run: echo "IMAGE_TAG=$GITHUB_SHA-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> $GITHUB_ENV && echo $IMAGE_TAG

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v2
        if: steps.prerun.outputs.result != 'skip'
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region:  us-west-2

      - name: Login to ECR
        if: ${{ steps.prerun.outputs.result != 'skip' }}
        uses: aws-actions/amazon-ecr-login@v1

      # Copy the production manifest.json to local directory - Raise error if not retrieved
      - name: Download dbt manifest from S3
        if: steps.prerun.outputs.result != 'skip' && github.event_name == 'pull_request'
        continue-on-error: true
        run: |
          output=$(aws s3 cp s3://hooli-demo/dbt_slim_ci/manifest.json ./dbt_project/target/slim_ci/manifest.json 2>&1) || echo "Error: $output"

      # Build 'data-eng-pipeline' code location
      - name: Build dbt manifest for data-eng-pipeline
        if: steps.prerun.outputs.result != 'skip'
        run: |
          pip install pyopenssl --upgrade;
          pip install click --upgrade;
          pip install dbt-core dbt-duckdb dbt-snowflake; 
          make deps
          make manifest

      # Copy production manifest.json to S3 on merge
      - name: Upload dbt manifest to S3
        if: steps.prerun.outputs.result != 'skip' && github.event_name != 'pull_request'
        run: |
          aws s3 cp ./dbt_project/target/manifest.json s3://hooli-demo/dbt_slim_ci/manifest.json

      - name: Build and upload Docker image for data-eng-pipeline
        if: steps.prerun.outputs.result != 'skip'
        uses: docker/build-push-action@v4
        with:
          context: .
          push: true
          tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-data-eng-pipeline

      - name: Update build session with image tag for data-eng-pipeline
        id: ci-set-build-output-data-eng-pipeline
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci set-build-output --location-name=data-eng-pipeline --image-tag=$IMAGE_TAG-data-eng-pipeline"

      # Build 'basics' code location
      - name: Build and upload Docker image for basics
        if: steps.prerun.outputs.result != 'skip'
        uses: docker/build-push-action@v4
        with:
          context: ./hooli_basics
          push: true
          tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-basics

      - name: Update build session with image tag for basics
        id: ci-set-build-output-basics
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci set-build-output --location-name=basics --image-tag=$IMAGE_TAG-basics"

      # Build 'batch enrichment' code location
      - name: Build and upload Docker image for batch enrichment
        if: steps.prerun.outputs.result != 'skip'
        uses: docker/build-push-action@v4
        with:
          context: ./hooli_batch_enrichment
          push: true
          tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-batch-enrichment

      - name: Update build session with image tag for batch enrichment
        id: ci-set-build-output-batch-enrichment
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci set-build-output --location-name=batch_enrichment --image-tag=$IMAGE_TAG-batch-enrichment"

      # Build 'snowflake_insights' code location
      - name: Build and upload Docker image for snowflake insights
        if: steps.prerun.outputs.result != 'skip'
        uses: docker/build-push-action@v4
        with:
          context: ./hooli_snowflake_insights
          push: true
          tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-snowflake-insights

      - name: Update build session with image tag for snowflake insights
        id: ci-set-build-output-snowflake-insights
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci set-build-output --location-name=snowflake_insights --image-tag=$IMAGE_TAG-snowflake-insights"

      # Build 'demo_assets' code location
      - name: Build and upload Docker image for demo_assets
        if: steps.prerun.outputs.result != 'skip'
        uses: docker/build-push-action@v4
        with:
          context: ./hooli-demo-assets
          push: true
          tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-demo-assets

      - name: Update build session with image tag for demo_assets
        id: ci-set-build-output-demo-assets
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci set-build-output --location-name=demo_assets --image-tag=$IMAGE_TAG-demo-assets"

      # Build pipes example container
      - name: Build and upload Docker image for pipes example
        if: steps.prerun.outputs.result != 'skip'
        uses: docker/build-push-action@v4
        with:
          context: ./hooli_data_eng/utils/example_container
          push: true
          tags: ${{ env.IMAGE_REGISTRY }}:latest-pipes-example

      # Deploy
      - name: Deploy to Dagster Cloud
        id: ci-deploy
        if: steps.prerun.outputs.result != 'skip'
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci deploy"

      # Get branch deployment as input to job trigger below
      - name: Get branch deployment
        id: get-branch-deployment
        if: steps.prerun.outputs.result != 'skip' && github.event_name == 'pull_request'
        uses: dagster-io/dagster-cloud-action/actions/utils/get_branch_deployment@v0.1
        with:
          organization_id: 'hooli'

      # Trigger dbt slim CI job
      - name: Trigger dbt slim CI
        if: steps.prerun.outputs.result != 'skip' && github.event_name == 'pull_request'
        uses: dagster-io/dagster-cloud-action/actions/utils/run@v0.1
        with:
          location_name: data-eng-pipeline
          deployment: ${{ steps.get-branch-deployment.outputs.deployment }}
          job_name: dbt_slim_ci_job
          organization_id: hooli

      # Summary and comment updates - note these always() run
      - name: Update PR comment for branch deployments
        id: ci-notify
        if: steps.prerun.outputs.result != 'skip' && always()
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}"

      - name: Generate summary
        id: ci-summary
        if: steps.prerun.outputs.result != 'skip' && always()
        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1.27
        with:
          command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY"