diff --git a/github/serverless/branch_deployments.yml b/github/serverless/branch_deployments.yml deleted file mode 100644 index 6a0f6dc..0000000 --- a/github/serverless/branch_deployments.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Serverless Branch Deployments -on: - pull_request: - types: [opened, synchronize, reopened, closed] - -concurrency: - # Cancel in-progress deploys to same branch - group: ${{ github.ref }}/branch_deployments - cancel-in-progress: true -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.8' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-20.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - - steps: - - name: Prerun Checks - id: prerun - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-20.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs - # env_vars: ${{ toJson(secrets) }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/github/serverless/deploy.yml b/github/serverless/deploy.yml index ab41916..22751d3 100644 --- a/github/serverless/deploy.yml +++ b/github/serverless/deploy.yml @@ -1,10 +1,12 @@ -name: Serverless Prod Deployment +name: Serverless Deployment on: push: branches: - "main" - "master" - + pull_request: + types: [opened, synchronize, reopened, closed] + concurrency: # Cancel in-progress deploys to same branch group: ${{ github.ref }}/deploy @@ -14,67 +16,100 @@ env: DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} ENABLE_FAST_DEPLOYS: 'true' PYTHON_VERSION: '3.8' + DAGSTER_PROJECT_DIR: '.' DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' + DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: dagster_cloud_default_deploy: name: Dagster Serverless Deploy runs-on: ubuntu-20.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - + steps: - name: Prerun Checks id: prerun uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' + - name: Checkout uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy + + # Validate dagster_cloud.yaml and the connection to dagster.cloud + - name: Validate configuration + id: ci-validate + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" + + # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session + - name: Initialize build session + id: ci-init + uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 + with: + project_dir: ${{ env.DAGSTER_PROJECT_DIR }} + dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} + # The full deployment name. If this run is for a PR, this value is ignored and a branch + # deployment is used. + deployment: 'prod' + + + # If using fast build, build the PEX + # First ensure the correct Python version is installed + - name: Set up Python ${{ env.PYTHON_VERSION }} for target + id: custom-python-version + if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install setuptools + if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} + run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools + shell: bash + + - name: Run PEX build + id: run-pex-build if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-20.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v3 + command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }} --pex-deps-cache-from='${{ github.repository }}' --pex-deps-cache-to='${{ github.repository }}'" + + + # Otherwise, enable buildx for caching and build the Docker image + - name: Set up Docker Buildx + if: steps.prerun.outputs.result == 'docker-deploy' + uses: docker/setup-buildx-action@v2 + + - name: Run Docker build + id: run-docker-build + if: steps.prerun.outputs.result == 'docker-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1 + command: "ci build --build-strategy=docker --python-version ${{ env.PYTHON_VERSION }}" + + + # Deploy all code locations in this build session to Dagster Cloud + - name: Deploy to Dagster Cloud + id: ci-deploy + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs - # env_vars: ${{ toJson(secrets) }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + command: "ci deploy" + # Update a PR comment - this runs always() so the comment is updated on success and failure + - name: Update PR comment for branch deployments + id: ci-notify + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}" + + # Generate a summary that shows up on the Workflow Summary page + - name: Generate a summary + id: ci-summary + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY"