Remove direct python 3.8 dependencies from dagster-cloud-action now t…

…hat it is EOL (#201)

Summary:
- Don't install python 3.8 in every serverless build unless that version is what is being used
- Use python 3.11 instead of 3.8 in the manylinux builder

Test Plan:
Create a release and run the action on a serverless deploy ofpython 3.8 through 3.12
Run a docker build locally using the latest manylinux builder

actions/build_deploy_python_executable/action.yml

@@ -11,7 +11,7 @@ inputs:
   deployment:
     required: false
     description: "The deployment to push to, defaults to 'prod'. Ignored for pull requests where the branch deployment is used."
-    default: "prod"  
+    default: "prod"
   deploy:
     description: 'Whether to upload the code files and update the code location'
     required: false
@@ -44,20 +44,14 @@ runs:
       run: echo "FLAG_DEPS_CACHE_TO=--deps-cache-to=${{ github.repository }}" >> $GITHUB_ENV
       shell: bash
 
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.8"
 
-    - if: ${{ inputs.python_version != '3.8' }}
-      id: custom-python-version
+    - id: setup-python
       name: Set up Python ${{ inputs.python_version }} for target
       uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python_version }}
 
-    - if: ${{ inputs.python_version != '3.8' }}
-      run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools
+    - run: ${{ steps.setup-python.outputs.python-path }} -m pip install setuptools
       shell: bash
 
     - run: >

github/serverless/dagster-plus-deploy.yml

@@ -15,7 +15,7 @@ env:
   DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
   DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
   ENABLE_FAST_DEPLOYS: 'true'
-  PYTHON_VERSION: '3.8'
+  PYTHON_VERSION: '3.10'
   DAGSTER_CLOUD_YAML_PATH: '.'
   DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
   DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }}
@@ -24,7 +24,7 @@ env:
 jobs:
   dagster_cloud_default_deploy:
     name: Dagster Serverless Deploy
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Prerun Checks
@@ -59,14 +59,14 @@ jobs:
       # If using fast build, build the PEX
       # First ensure the correct Python version is installed
       - name: Set up Python ${{ env.PYTHON_VERSION }} for target
-        id: custom-python-version
-        if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }}
+        id: setup-python-version
+        if: steps.pre-run.outputs.result == 'pex-deploy'
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Install setuptools
-        if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }}
-        run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools
+        if: steps.pre-run.outputs.result == 'pex-deploy'
+        run: ${{ steps.setup-python-version.outputs.python-path }} -m pip install setuptools
         shell: bash
 
       - name: Run PEX build

github/serverless/dbt/branch_deployments.yml

@@ -12,14 +12,14 @@ env:
   DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }}
   DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
   ENABLE_FAST_DEPLOYS: 'true'
-  PYTHON_VERSION: '3.8'
+  PYTHON_VERSION: '3.10'
   DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
   DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'
 
 jobs:
   dagster_cloud_default_deploy:
     name: Dagster Serverless Deploy
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       build_info: ${{ steps.parse-workspace.outputs.build_info }}
 
@@ -84,7 +84,7 @@ jobs:
 
   dagster_cloud_docker_deploy:
     name: Docker Deploy
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: needs.dagster_cloud_default_deploy.outputs.build_info
     needs: dagster_cloud_default_deploy
     strategy:

github/serverless/dbt/deploy.yml

@@ -14,14 +14,14 @@ env:
   DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }}
   DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
   ENABLE_FAST_DEPLOYS: 'true'
-  PYTHON_VERSION: '3.8'
+  PYTHON_VERSION: '3.10'
   DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
   DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'
 
 jobs:
   dagster_cloud_default_deploy:
     name: Dagster Serverless Deploy
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       build_info: ${{ steps.parse-workspace.outputs.build_info }}
 
@@ -84,7 +84,7 @@ jobs:
 
   dagster_cloud_docker_deploy:
     name: Docker Deploy
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: needs.dagster_cloud_default_deploy.outputs.build_info
     needs: dagster_cloud_default_deploy
     strategy:

gitlab/dbt/serverless-ci-dbt.yml

@@ -5,8 +5,8 @@ variables:
   DAGSTER_DBT_PROJECT_NAME: $DAGSTER_DBT_LOCATION_NAME
   DAGSTER_DBT_PROJECT_DIR: "${CI_PROJECT_DIR}"
   DAGSTER_DBT_PACKAGE_DATA_DIR: "${CI_PROJECT_DIR}/$DAGSTER_DBT_PROJECT_NAME/dbt-project"
-  # Python versions 3.8 to 3.11 are supported
-  PYTHON_VERSION: '3.8'
+  # Python versions 3.8 to 3.12 are supported
+  PYTHON_VERSION: '3.10'
 
 deploy-branch:
   stage: deploy

gitlab/serverless-ci.yml

@@ -2,8 +2,8 @@ variables:
   DISABLE_FAST_DEPLOYS:
   DAGSTER_CLOUD_URL: $DAGSTER_CLOUD_URL
   DAGSTER_CLOUD_API_TOKEN: $DAGSTER_CLOUD_API_TOKEN
-  # Python versions 3.8 to 3.11 are supported
-  PYTHON_VERSION: '3.8'
+  # Python versions 3.8 to 3.12 are supported
+  PYTHON_VERSION: '3.10'
 
 deploy-branch:
   stage: deploy

scripts/release.py

@@ -128,8 +128,7 @@ def update_dagster_cloud_pex(
         "--pip-version=23.0",
         "--resolver-version=pip-2020-resolver",
         "--venv=prepend",
-        # use a /bin/sh entrypoint that is better at choosing a python interpreter to use
-        "--sh-boot",
+        "--python-shebang=/usr/bin/env python",
         "-v",
     ]
     print(f"Running {args}")

src/Dockerfile.dagster-manylinux-builder

@@ -1,6 +1,6 @@
 # Builds ghcr.io/daster-io/dagster-manylinux-builder:*
 
-# This docker image contains the PEX builder (builder.pex) and is capable of 
+# This docker image contains the PEX builder (builder.pex) and is capable of
 # building source only dependencies (sdists) for Python that work with Dagster Cloud Serverless base
 # images.
 
@@ -12,8 +12,8 @@
 #
 # $ echo $GITHUB-PAT | docker login ghcr.io -u USERNAME --password-stdin
 #
-# $ docker build . -f src/Dockerfile.dagster-manylinux-builder -t ghcr.io/dagster-io/dagster-manylinux-builder:latest
-# $ docker push ghcr.io/dagster-io/dagster-manylinux-builder:latest
+# $ docker build . -f src/Dockerfile.dagster-manylinux-builder -t ghcr.io/dagster-io/dagster-manylinux-builder:YOUR_IMAGE_TAG_HERE
+# $ docker push ghcr.io/dagster-io/dagster-manylinux-builder:YOUR_IMAGE_TAG_HERE
 
 # See dagster-cloud (docker_runner.py) for code that uses this image.
 
@@ -23,13 +23,13 @@
 FROM --platform=linux/amd64 quay.io/pypa/manylinux_2_28_x86_64:latest
 
 # Add all the relevant Python binaries to the PATH
-ENV PATH="/opt/python/cp38-cp38/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:$PATH"
+ENV PATH="/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:/opt/python/cp311-cp312/bin:$PATH"
 
 # To install unreleased versions, build the wheels and drop them in this directory
 COPY wheels /wheels
 
 # Install dagster-cloud
-RUN python3.8 -m pip install dagster-cloud --find-links file:///wheels/
+RUN python3.11 -m pip install dagster-cloud --find-links file:///wheels/
 
 COPY generated/gha/builder.pex /builder.pex