Build dagster-cloud-action PEX for more platforms

Summary: Make our dagster-cloud-cli work on github action runners that have python versions other than 3.8 installed.
dagster-io · Oct 2, 2024 · 8f422f4 · 8f422f4
1 parent 468ba21
commit 8f422f4
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 16 deletions.
diff --git a/scripts/release.py b/scripts/release.py
@@ -104,6 +104,16 @@ def update_dagster_cloud_pex(
         info("Using PyPI for dagster package")
         dagster_pkg = "dagster"
 
+    platform_args = []
+
+    # each of the default versions used by ubuntu 20.04 / 22.04 / 24.04 respectively
+    for py_version in ["38", "310", "312"]:
+        platform_args.extend(
+            [
+                f"--platform=manylinux2014_x86_64-cp-{py_version}-cp{py_version}",
+            ]
+        )
+
     info("Building generated/gha/dagster-cloud.pex")
     args = [
         "pex",
@@ -112,12 +122,14 @@ def update_dagster_cloud_pex(
         dagster_pkg,
         "PyGithub",
         "-o=dagster-cloud.pex",
-        "--platform=manylinux2014_x86_64-cp-38-cp38",
-        "--platform=macosx_12_0_x86_64-cp-38-cp38",
-        "--platform=macosx_12_0_arm64-cp-38-cp38",
+        *platform_args,
+        "--platform=macosx_12_0_x86_64-cp-311-cp311",
+        "--platform=macosx_12_0_arm64-cp-311-cp311",
         "--pip-version=23.0",
         "--resolver-version=pip-2020-resolver",
         "--venv=prepend",
+        # use a /bin/sh entrypoint that is better at choosing a python interpreter to use
+        "--sh-boot",
         "-v",
     ]
     print(f"Running {args}")

diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py
@@ -2,12 +2,7 @@
 import subprocess
 import tempfile
 from contextlib import contextmanager
-from pathlib import Path
 from typing import List
-from unittest import mock
-
-import pytest
-import requests
 
 
 @contextmanager
@@ -23,18 +18,16 @@ def run_dagster_cloud_serverless_cmd(dagster_cloud_pex_path, args: List[str]):
                 build_output_dir,
             ],
             capture_output=True,
+            check=False,
         )
         if proc.returncode:
             raise ValueError(
-                "Failed to run dagster-cloud.pex:"
-                + (proc.stdout + proc.stderr).decode("utf-8")
+                "Failed to run dagster-cloud.pex:" + (proc.stdout + proc.stderr).decode("utf-8")
             )
 
         all_files = os.listdir(build_output_dir)
         pex_files = {
-            filename
-            for filename in all_files
-            if filename.endswith(".pex") and filename != ".pex"
+            filename for filename in all_files if filename.endswith(".pex") and filename != ".pex"
         }
         yield (build_output_dir, list(pex_files), list(set(all_files) - pex_files))
 
@@ -48,6 +41,7 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path):
             str(dagster_project1),
             "--api-token=fake",
             "--url=fake",
+            "--python-version=3.11",
         ],
     ) as (
         build_output_dir,
@@ -56,9 +50,7 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path):
     ):
         # one source-HASH.pex and one deps-HASH.pex file are expected
         assert 2 == len(pex_files)
-        pex_file_by_alias = {
-            filename.split("-", 1)[0]: filename for filename in pex_files
-        }
+        pex_file_by_alias = {filename.split("-", 1)[0]: filename for filename in pex_files}
 
         assert {"source", "deps"} == set(pex_file_by_alias)