From ab69376dd14809d94bfb6e93b7fd0cd781a0ff1b Mon Sep 17 00:00:00 2001 From: Ben Pankow Date: Mon, 23 Sep 2024 15:24:19 -0700 Subject: [PATCH] Introduce individual package publish Buildkite script (#24616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary & Motivation Introduces a new BK pipeline which we can manually trigger to publish a package outside of the typical release process. Right now, pulls a list of possible packages that do not have dynamic versions in `setup.py`: ![Screenshot 2024-09-19 at 4 08 03 PM](https://github.com/user-attachments/assets/4469c97c-d2c4-49ae-b9eb-d5887bb9a097) The main use-case is for new integrations and experimental packages that we may want to iterate on rapidly, e.g. release more than once a week, separate from our stable package release process. The pipeline: 1. Updates the version tag in code 2. Builds and publishes the package to pypi 3. Commits the new version tag 4. Pushes to a git tag `{package_name}/v{version}` and the selected branch on GitHub ## How I Tested These Changes Tested manually in BK ## Changelog NOCHANGELOG --- .../dagster_buildkite/cli.py | 8 ++ .../pipelines/prerelease_package.py | 70 +++++++++++++ .../dagster_buildkite/steps/packages.py | 2 +- .../dagster_buildkite/utils.py | 39 +++++++- .buildkite/dagster-buildkite/setup.py | 1 + .../libraries/dagster-powerbi/setup.py | 14 +-- .../dagster-sigma/dagster_sigma/__init__.py | 2 +- .../libraries/dagster-sigma/setup.py | 15 +-- scripts/build_and_publish.sh | 99 ++++++------------- 9 files changed, 152 insertions(+), 98 deletions(-) create mode 100644 .buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/cli.py b/.buildkite/dagster-buildkite/dagster_buildkite/cli.py index 85b547a66f132..2b80a634ad8d7 100755 --- a/.buildkite/dagster-buildkite/dagster_buildkite/cli.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/cli.py @@ -3,6 +3,7 @@ from dagster_buildkite.git import GitInfo from dagster_buildkite.pipelines.dagster_oss_main import build_dagster_oss_main_steps from dagster_buildkite.pipelines.dagster_oss_nightly_pipeline import build_dagster_oss_nightly_steps +from dagster_buildkite.pipelines.prerelease_package import build_prerelease_package_steps from dagster_buildkite.python_packages import PythonPackages from dagster_buildkite.utils import buildkite_yaml_for_steps @@ -24,3 +25,10 @@ def dagster_nightly() -> None: steps = build_dagster_oss_nightly_steps() buildkite_yaml = buildkite_yaml_for_steps(steps, custom_slack_channel="eng-buildkite-nightly") print(buildkite_yaml) # noqa: T201 + + +def prerelease_package() -> None: + PythonPackages.load_from_git(GitInfo(directory=Path("."))) + steps = build_prerelease_package_steps() + buildkite_yaml = buildkite_yaml_for_steps(steps) + print(buildkite_yaml) # noqa: T201 diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py b/.buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py new file mode 100644 index 0000000000000..7bda9c847b57d --- /dev/null +++ b/.buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py @@ -0,0 +1,70 @@ +import re +from pathlib import Path +from typing import List + +from dagster_buildkite.python_version import AvailablePythonVersion +from dagster_buildkite.step_builder import CommandStepBuilder +from dagster_buildkite.steps.packages import _get_uncustomized_pkg_roots +from dagster_buildkite.utils import BlockStep, BuildkiteStep + + +def build_prerelease_package_steps() -> List[BuildkiteStep]: + steps: List[BuildkiteStep] = [] + + packages = ( + _get_uncustomized_pkg_roots("python_modules", []) + + _get_uncustomized_pkg_roots("python_modules/libraries", []) + + _get_uncustomized_pkg_roots("examples/experimental", []) + ) + + # Get only packages that have a fixed version in setup.py + filtered_packages = [] + for package in packages: + setup_file = Path(package) / "setup.py" + contents = setup_file.read_text() + if re.findall(r"version=\"[\d\.]+\"", contents): + filtered_packages.append(package) + + input_step: BlockStep = { + "block": ":question: Choose package", + "prompt": None, + "fields": [ + { + "select": "Select a package to publish", + "key": "package-to-release-path", + "options": [ + { + "label": package[len("python_modules/") :] + if package.startswith("python_modules/") + else package, + "value": package, + } + for package in filtered_packages + ], + "hint": None, + "default": None, + "required": True, + "multiple": None, + }, + { + "text": "Enter the version to publish", + "required": False, + "key": "version-to-release", + "default": None, + "hint": "Leave blank to auto-increment the minor version", + }, + ], + } + steps.append(input_step) + + steps.append( + CommandStepBuilder(":package: Build and publish package") + .run( + "pip install build", + "sh ./scripts/build_and_publish.sh", + ) + .on_test_image(AvailablePythonVersion.get_default(), env=["PYPI_TOKEN"]) + .build() + ) + + return steps diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py index e2f2c0fb23679..d0cbed993a66d 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py @@ -74,7 +74,7 @@ def build_steps_from_package_specs(package_specs: List[PackageSpec]) -> List[Bui # Find packages under a root subdirectory that are not configured above. -def _get_uncustomized_pkg_roots(root, custom_pkg_roots) -> List[str]: +def _get_uncustomized_pkg_roots(root: str, custom_pkg_roots: List[str]) -> List[str]: all_files_in_root = [ os.path.relpath(p, GIT_REPO_ROOT) for p in glob(os.path.join(GIT_REPO_ROOT, root, "*")) ] diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/utils.py b/.buildkite/dagster-buildkite/dagster_buildkite/utils.py index 93c20502c30f8..2b6cb869423ed 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/utils.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/utils.py @@ -63,7 +63,40 @@ class GroupStep(TypedDict): WaitStep: TypeAlias = Literal["wait"] -BuildkiteStep: TypeAlias = Union[CommandStep, GroupStep, TriggerStep, WaitStep] +InputSelectOption = TypedDict("InputSelectOption", {"label": str, "value": str}) +InputSelectField = TypedDict( + "InputSelectField", + { + "select": str, + "key": str, + "options": List[InputSelectOption], + "hint": Optional[str], + "default": Optional[str], + "required": Optional[bool], + "multiple": Optional[bool], + }, +) +InputTextField = TypedDict( + "InputTextField", + { + "text": str, + "key": str, + "hint": Optional[str], + "default": Optional[str], + "required": Optional[bool], + }, +) + +BlockStep = TypedDict( + "BlockStep", + { + "block": str, + "prompt": Optional[str], + "fields": List[Union[InputSelectField, InputTextField]], + }, +) + +BuildkiteStep: TypeAlias = Union[CommandStep, GroupStep, TriggerStep, WaitStep, BlockStep] BuildkiteLeafStep = Union[CommandStep, TriggerStep, WaitStep] BuildkiteTopLevelStep = Union[CommandStep, GroupStep] @@ -84,7 +117,9 @@ def safe_getenv(env_var: str) -> str: return os.environ[env_var] -def buildkite_yaml_for_steps(steps, custom_slack_channel: Optional[str] = None) -> str: +def buildkite_yaml_for_steps( + steps: Sequence[BuildkiteStep], custom_slack_channel: Optional[str] = None +) -> str: return yaml.dump( { "env": { diff --git a/.buildkite/dagster-buildkite/setup.py b/.buildkite/dagster-buildkite/setup.py index 1e8293f2988e4..4a82775081451 100644 --- a/.buildkite/dagster-buildkite/setup.py +++ b/.buildkite/dagster-buildkite/setup.py @@ -30,6 +30,7 @@ "console_scripts": [ "dagster-buildkite = dagster_buildkite.cli:dagster", "dagster-buildkite-nightly = dagster_buildkite.cli:dagster_nightly", + "dagster-buildkite-prerelease-package = dagster_buildkite.cli:prerelease_package", ] }, ) diff --git a/python_modules/libraries/dagster-powerbi/setup.py b/python_modules/libraries/dagster-powerbi/setup.py index 8456fa1cbfeb7..7f13982ad97b6 100644 --- a/python_modules/libraries/dagster-powerbi/setup.py +++ b/python_modules/libraries/dagster-powerbi/setup.py @@ -1,22 +1,10 @@ from setuptools import find_packages, setup - -def get_version() -> str: - return "1!0+dev" - # Uncomment when ready to publish - # version: Dict[str, str] = {} - # with open(Path(__file__).parent / "dagster_powerbi/version.py", encoding="utf8") as fp: - # exec(fp.read(), version) - - # return version["__version__"] - - -ver = get_version() # dont pin dev installs to avoid pip dep resolver issues pin = "" setup( name="dagster_powerbi", - version=get_version(), + version="0.0.3", author="Dagster Labs", author_email="hello@dagsterlabs.com", license="Apache-2.0", diff --git a/python_modules/libraries/dagster-sigma/dagster_sigma/__init__.py b/python_modules/libraries/dagster-sigma/dagster_sigma/__init__.py index b37f4eab674db..1d028c452cd90 100644 --- a/python_modules/libraries/dagster-sigma/dagster_sigma/__init__.py +++ b/python_modules/libraries/dagster-sigma/dagster_sigma/__init__.py @@ -11,6 +11,6 @@ ) # Move back to version.py and edit setup.py once we are ready to publish. -__version__ = "1!0+dev" +__version__ = "0.0.10" DagsterLibraryRegistry.register("dagster-sigma", __version__) diff --git a/python_modules/libraries/dagster-sigma/setup.py b/python_modules/libraries/dagster-sigma/setup.py index 0b53c980cee75..7c5ee84127f4a 100644 --- a/python_modules/libraries/dagster-sigma/setup.py +++ b/python_modules/libraries/dagster-sigma/setup.py @@ -1,22 +1,9 @@ from setuptools import find_packages, setup - -def get_version() -> str: - return "1!0+dev" - # Uncomment when ready to publish - # version: Dict[str, str] = {} - # with open(Path(__file__).parent / "dagster_sigma/version.py", encoding="utf8") as fp: - # exec(fp.read(), version) - - # return version["__version__"] - - -ver = get_version() -# dont pin dev installs to avoid pip dep resolver issues pin = "" setup( name="dagster_sigma", - version=get_version(), + version="0.0.10", author="Dagster Labs", author_email="hello@dagsterlabs.com", license="Apache-2.0", diff --git a/scripts/build_and_publish.sh b/scripts/build_and_publish.sh index 324b73353c7d0..a523b1abbfcdb 100755 --- a/scripts/build_and_publish.sh +++ b/scripts/build_and_publish.sh @@ -1,86 +1,51 @@ -# How to release: -# 2. ensure you have an API key from the elementl PyPI account (account is in the password manager) -# 3. run make adhoc_pypi from the root of the dagster-airlift directory -# 4. once propted, use '__token__' for the username and the API key for the password +PACKAGE_TO_RELEASE_PATH=$(buildkite-agent meta-data get package-to-release-path) +VERSION_TO_RELEASE=$(buildkite-agent meta-data get version-to-release --default '') -# Define the path to the .pypirc file -PYPIRC_FILE="$HOME/.pypirc" - -PACKAGE_TO_RELEASE_PATH=$1 -VERSION_TO_RELEASE=$2 +git checkout $BUILDKITE_BRANCH if [ -z "$PACKAGE_TO_RELEASE_PATH" ]; then echo "Please provide the path to the package to release." exit 1 fi if [ -z "$VERSION_TO_RELEASE" ]; then - echo "Please provide the version to release." - exit 1 -fi - -# Define cleanup function -cleanup() { - echo "Cleaning up..." - rm -rf dist/* -} - -# Set trap to call cleanup function on script exit -trap cleanup EXIT + echo "Inferring version to release from package." + EXISTING_VERSION=$(grep 'version=' $PACKAGE_TO_RELEASE_PATH/setup.py) + echo "Existing version: $EXISTING_VERSION" + MAJOR_VAR=$(echo $EXISTING_VERSION | sed -E 's/.*version=[^0-9]([0-9].+)([0-9]+).*/\1/') + MINOR_VAR=$(echo $EXISTING_VERSION | sed -E 's/.*version=[^0-9]([0-9].+)([0-9]+).*/\2/') + INCREMENTED_MINOR_VAR=$((MINOR_VAR + 1)) -# Check if the .pypirc file exists -if [ ! -f "$PYPIRC_FILE" ]; then - echo ".pypirc file not found in $HOME." + VERSION_TO_RELEASE="$MAJOR_VAR$INCREMENTED_MINOR_VAR" - # Prompt the user for the API token - read -p "Enter your API token (must start with 'pypi-'): " API_TOKEN - - # Check if the API token starts with 'pypi-' - if [[ $API_TOKEN != pypi-* ]]; then - echo "Invalid API token. It must start with 'pypi-'." - exit 1 - fi - - # Create the .pypirc file and write the configuration - cat < "$PYPIRC_FILE" -[pypi] -username = __token__ -password = $API_TOKEN -EOF - - echo ".pypirc file created successfully." -else - echo ".pypirc file already exists in $HOME. Using that as pypi credentials." + echo "Going to release version $VERSION_TO_RELEASE" fi -rm -rf dist/* -rm -rf package_prerelease -mkdir -p package_prerelease -cp -R $PACKAGE_TO_RELEASE_PATH/* package_prerelease -pushd package_prerelease - # Update both a hardcoded version, if set, in setup.py, and # find where __version__ is set and update it -sed -i "" "s|return \"1!0+dev\"|return \"$VERSION_TO_RELEASE\"|" setup.py -grep -rl "__version__ = \"1!0+dev\"" ./ | xargs sed -i "" "s|\"1!0+dev\"|\"$VERSION_TO_RELEASE\"|" +echo "Updating version in source..." +sed -i "s|version=\".*\"|version=\"$VERSION_TO_RELEASE\"|" "$PACKAGE_TO_RELEASE_PATH/setup.py" +grep -rl "__version__ = \".*\"" "$PACKAGE_TO_RELEASE_PATH" | xargs sed -i "s|__version__ = \".*\"|__version__ = \"$VERSION_TO_RELEASE\"|" + +mkdir -p package_prerelease +cp -R $PACKAGE_TO_RELEASE_PATH/* package_prerelease +cd package_prerelease echo "Building package..." python3 -m build + echo "Uploading to pypi..." -# Capture the output of the twine upload command -TWINE_OUTPUT=$(python3 -m twine upload --repository pypi dist/* --verbose 2>&1) -TWINE_EXIT_CODE=$? +python3 -m twine upload --username "__token__" --password "$PYPI_TOKEN" --repository pypi dist/* --verbose -# Check if the output contains a 400 error -if echo "$TWINE_OUTPUT" | grep -q "400 Bad Request"; then - echo "Error: Twine upload failed with a 400 Bad Request error." - echo "Twine output:" - echo "$TWINE_OUTPUT" - exit 1 -elif [ $TWINE_EXIT_CODE -ne 0 ]; then - echo "Error: Twine upload failed with exit code $TWINE_EXIT_CODE." - echo "Twine output:" - echo "$TWINE_OUTPUT" - exit $TWINE_EXIT_CODE -fi +cd .. +rm -rf package_prerelease + +echo "Committing and tagging release..." +PACKAGE_NAME=$(echo $PACKAGE_TO_RELEASE_PATH | awk -F/ '{print $NF}') +git add -A +git config --global user.email "devtools@dagsterlabs.com" +git config --global user.name "Dagster Labs" +git commit -m "$PACKAGE_NAME $VERSION_TO_RELEASE" -echo "Upload successful." +git tag "$PACKAGE_NAME/v$VERSION_TO_RELEASE" +git push origin "$PACKAGE_NAME/v$VERSION_TO_RELEASE" +git push origin $BUILDKITE_BRANCH