From 3f03263911e582d94bf2320b599f56da02bae824 Mon Sep 17 00:00:00 2001 From: gibsondan Date: Thu, 31 Oct 2024 11:50:25 -0500 Subject: [PATCH] test progress --- scripts/Dockerfile.dagster-cloud-pex-builder | 6 +- scripts/release.py | 4 +- src/Dockerfile.test-dagster-cloud-pex | 6 + tests/test_pex_builder.py | 109 ++++++++++++------- 4 files changed, 79 insertions(+), 46 deletions(-) create mode 100644 src/Dockerfile.test-dagster-cloud-pex diff --git a/scripts/Dockerfile.dagster-cloud-pex-builder b/scripts/Dockerfile.dagster-cloud-pex-builder index f46cd37..9748453 100644 --- a/scripts/Dockerfile.dagster-cloud-pex-builder +++ b/scripts/Dockerfile.dagster-cloud-pex-builder @@ -1,12 +1,10 @@ -# Builds generated/gha/dagster-cloud.pex +# Builds generated/gha/dagster-cloud.pex and writes it to /generated # Use an official manylinux builder (https://github.com/pypa/manylinux#docker-images) FROM --platform=linux/amd64 quay.io/pypa/manylinux_2_28_x86_64:latest -# Add all the relevant Python binaries to the PATH -ENV PATH="/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin:/opt/python/cp311-cp311/bin:/opt/python/cp311-cp312/bin:$PATH" +ENV PATH="/opt/python/cp311-cp311/bin:$PATH" -# pins? RUN python3.11 -m pip install typer rich pex COPY release.py release.py diff --git a/scripts/release.py b/scripts/release.py index cecf4f6..8b8f5e5 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -81,7 +81,7 @@ def build_docker_action(version_tag: str, publish_docker_action: bool = True): print(output) -@app.command(help="Build dagster-cloud.pex") +@app.command(help="Build dagster-cloud.pex - invoked by the dagster-cloud-pex-builder image") def build_dagster_cloud_pex( dagster_internal_branch: Optional[str] = DAGSTER_INTERNAL_BRANCH_OPTION, dagster_oss_branch: Optional[str] = DAGSTER_OSS_BRANCH_OPTION, @@ -147,6 +147,7 @@ def update_dagster_cloud_pex( dagster_internal_branch: Optional[str] = DAGSTER_INTERNAL_BRANCH_OPTION, dagster_oss_branch: Optional[str] = DAGSTER_OSS_BRANCH_OPTION, ): + # Map /generated on the docker image to our local generated folder map_folders = {"/generated": os.path.join(os.path.dirname(__file__), "..", "generated")} mount_args = [] @@ -167,7 +168,6 @@ def update_dagster_cloud_pex( subprocess.run(cmd, check=True) - # Map generated to generated on docker image and build! cmd = [ "docker", "run", diff --git a/src/Dockerfile.test-dagster-cloud-pex b/src/Dockerfile.test-dagster-cloud-pex new file mode 100644 index 0000000..576b134 --- /dev/null +++ b/src/Dockerfile.test-dagster-cloud-pex @@ -0,0 +1,6 @@ +# Used for testing the generated dagster-cloud pex +FROM --platform=linux/amd64 python:3.10-slim + +COPY generated/gha/dagster-cloud.pex /dagster-cloud.pex + +ENTRYPOINT ["/dagster-cloud.pex"] diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index e778293..7e130d5 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -1,53 +1,74 @@ import os import subprocess import tempfile -from contextlib import contextmanager -from typing import List +from typing import List, Mapping +import pytest + + +def run_dagster_cloud_serverless_cmd(args: List[str], map_folders: Mapping[str, str]): + mount_args = [] + for target_folder, source_folder in map_folders.items(): + mount_args.extend(["--mount", f"type=bind,source={source_folder},target={target_folder}"]) + + cmd = [ + "docker", + "run", + "--platform=linux/amd64", + *mount_args, + "-t", + "test-dagster-cloud-pex", + "-m", + "dagster_cloud_cli.entrypoint", + "serverless", + *args, + ] + + subprocess.run(cmd, encoding="utf-8", capture_output=False, check=True) + + +@pytest.fixture +def built_test_dagster_cloud_pex_image(repo_root: str): + src_dir = os.path.join(os.path.dirname(__file__), "..", "src") + + cmd = [ + "docker", + "build", + "--progress=plain", + "-t", + "test-dagster-cloud-pex", + "--platform=linux/amd64", + "-f", + os.path.join(src_dir, "Dockerfile.test-dagster-cloud-pex"), + repo_root, + ] + + subprocess.run(cmd, check=True) + + +def test_pex_build_only(repo_root, built_test_dagster_cloud_pex_image): + dagster_project1 = repo_root / "tests/test-repos/dagster_project1" -@contextmanager -def run_dagster_cloud_serverless_cmd(dagster_cloud_pex_path, args: List[str]): with tempfile.TemporaryDirectory() as build_output_dir: - proc = subprocess.run( + map_folders = {"/dagster_project1": dagster_project1, "/build_output_dir": build_output_dir} + + run_dagster_cloud_serverless_cmd( [ - dagster_cloud_pex_path, - "-m", - "dagster_cloud_cli.entrypoint", - "serverless", - *args, - build_output_dir, + "build-python-executable", + "/dagster_project1", + "--api-token=fake", + "--url=fake", + "--python-version=3.10", + "/build_output_dir", ], - capture_output=True, - check=False, + map_folders=map_folders, ) - if proc.returncode: - raise ValueError( - "Failed to run dagster-cloud.pex:" + (proc.stdout + proc.stderr).decode("utf-8") - ) all_files = os.listdir(build_output_dir) pex_files = { filename for filename in all_files if filename.endswith(".pex") and filename != ".pex" } - yield (build_output_dir, list(pex_files), list(set(all_files) - pex_files)) - -def test_pex_build_only(repo_root, dagster_cloud_pex_path): - dagster_project1 = repo_root / "tests/test-repos/dagster_project1" - with run_dagster_cloud_serverless_cmd( - dagster_cloud_pex_path, - [ - "build-python-executable", - str(dagster_project1), - "--api-token=fake", - "--url=fake", - "--python-version=3.11", - ], - ) as ( - build_output_dir, - pex_files, - other_files, - ): # one source-HASH.pex and one deps-HASH.pex file are expected assert 2 == len(pex_files) pex_file_by_alias = {filename.split("-", 1)[0]: filename for filename in pex_files} @@ -55,8 +76,16 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path): assert {"source", "deps"} == set(pex_file_by_alias) -def test_dagster_cloud_runnable(dagster_cloud_pex_path): - output = subprocess.check_output( - [dagster_cloud_pex_path, "-c", "print('hello')"], encoding="utf-8" - ) - assert "hello" in output +def test_dagster_cloud_runnable(built_test_dagster_cloud_pex_image): + cmd = [ + "docker", + "run", + "--platform=linux/amd64", + "-t", + "test-dagster-cloud-pex", + "-c", + "print('hello')", + ] + output = subprocess.run(cmd, encoding="utf-8", capture_output=True, check=True) + + assert "hello" in output.stdout