From 69e4fd803fd80dc43284f66b3368a20af319a0b1 Mon Sep 17 00:00:00 2001 From: Emily Voss Date: Thu, 1 May 2025 10:43:23 -0700 Subject: [PATCH 1/3] Improved versioning --- .github/workflows/docker-publish.yml | 6 +++++- Makefile | 2 -- prepline_general/__version__.py | 1 + prepline_general/api/general.py | 5 +++-- preprocessing-pipeline-family.yaml | 2 -- 5 files changed, 9 insertions(+), 7 deletions(-) create mode 100644 prepline_general/__version__.py delete mode 100644 preprocessing-pipeline-family.yaml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 60006796..008fe5a6 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -131,6 +131,11 @@ jobs: docker push $DOCKER_BUILD_REPOSITORY:amd64 #docker tag $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:arm64 #docker push $DOCKER_BUILD_REPOSITORY:arm64 + - name: Extract version from __version__.py + id: extract-version + run: | + VERSION=$(grep -oP "(?<=^__version__ = ')[^']+" prepline_general/__version__.py) + echo "VERSION=$VERSION" >> $GITHUB_ENV - name: Push multiarch manifest run: | #docker manifest create ${DOCKER_REPOSITORY}:latest $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64 @@ -139,7 +144,6 @@ jobs: #docker manifest create ${DOCKER_REPOSITORY}:$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64 docker manifest create ${DOCKER_REPOSITORY}:$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64 docker manifest push $DOCKER_REPOSITORY:$SHORT_SHA - VERSION=$(grep -m1 version preprocessing-pipeline-family.yaml | cut -d ' ' -f2) #docker manifest create ${DOCKER_REPOSITORY}:$VERSION $DOCKER_BUILD_REPOSITORY:amd64 $DOCKER_BUILD_REPOSITORY:arm64 docker manifest create ${DOCKER_REPOSITORY}:$VERSION $DOCKER_BUILD_REPOSITORY:amd64 docker manifest push ${DOCKER_REPOSITORY}:$VERSION diff --git a/Makefile b/Makefile index cd806e12..a1119fa5 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,6 @@ check-version: # Fail if syncing version would produce changes scripts/version-sync.sh -c \ -s CHANGELOG.md \ - -f preprocessing-pipeline-family.yaml release \ -f ${PACKAGE_NAME}/api/app.py release \ -f ${PACKAGE_NAME}/api/general.py release @@ -145,6 +144,5 @@ check-version: version-sync: scripts/version-sync.sh \ -s CHANGELOG.md \ - -f preprocessing-pipeline-family.yaml release \ -f ${PACKAGE_NAME}/api/app.py release \ -f ${PACKAGE_NAME}/api/general.py release diff --git a/prepline_general/__version__.py b/prepline_general/__version__.py new file mode 100644 index 00000000..be93fd37 --- /dev/null +++ b/prepline_general/__version__.py @@ -0,0 +1 @@ +__version__ = "0.0.85" # pragma: no cover diff --git a/prepline_general/api/general.py b/prepline_general/api/general.py index 4d4b563e..492ab47a 100644 --- a/prepline_general/api/general.py +++ b/prepline_general/api/general.py @@ -33,6 +33,7 @@ from starlette.datastructures import Headers from starlette.types import Send +from prepline_general.__version__ import __version__ as API_VERSION from prepline_general.api.models.form_params import GeneralFormParams from prepline_general.api.filetypes import get_validated_mimetype from unstructured.documents.elements import Element @@ -602,7 +603,7 @@ def return_content_type(filename: str): @router.get("/general/v0/general", include_in_schema=False) -@router.get("/general/v0.0.84/general", include_in_schema=False) +@router.get(f"/general/{API_VERSION}/general", include_in_schema=False) async def handle_invalid_get_request(): raise HTTPException( status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Only POST requests are supported." @@ -617,7 +618,7 @@ async def handle_invalid_get_request(): description="Description", operation_id="partition_parameters", ) -@router.post("/general/v0.0.84/general", include_in_schema=False) +@router.post(f"/general/{API_VERSION}/general", include_in_schema=False) def general_partition( request: Request, # cannot use annotated type here because of a bug described here: diff --git a/preprocessing-pipeline-family.yaml b/preprocessing-pipeline-family.yaml deleted file mode 100644 index acdfdd37..00000000 --- a/preprocessing-pipeline-family.yaml +++ /dev/null @@ -1,2 +0,0 @@ -name: general -version: 0.0.84 From f637d04f72a541ce8a8595fecd29f90fa2de2515 Mon Sep 17 00:00:00 2001 From: Emily Voss Date: Thu, 1 May 2025 10:59:04 -0700 Subject: [PATCH 2/3] Remove version sync as we no longer need it --- .github/workflows/bump_libraries.yaml | 1 - .github/workflows/docker-publish.yml | 2 +- Makefile | 17 --- scripts/version-sync.sh | 154 -------------------------- 4 files changed, 1 insertion(+), 173 deletions(-) delete mode 100755 scripts/version-sync.sh diff --git a/.github/workflows/bump_libraries.yaml b/.github/workflows/bump_libraries.yaml index 79c25615..a3a7128e 100644 --- a/.github/workflows/bump_libraries.yaml +++ b/.github/workflows/bump_libraries.yaml @@ -37,7 +37,6 @@ jobs: package=${package%\[*} changelog_message="Bump $package to ${{ steps.metadata.outputs.new-version }}" ./scripts/version-increment.sh "$changelog_message" - make version-sync - uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: "Bump libraries and release" diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 008fe5a6..ad57dfbe 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -134,7 +134,7 @@ jobs: - name: Extract version from __version__.py id: extract-version run: | - VERSION=$(grep -oP "(?<=^__version__ = ')[^']+" prepline_general/__version__.py) + VERSION=$(python3 -c "with open('prepline_general/__version__.py') as f: exec(f.read()); print(__version__)") echo "VERSION=$VERSION" >> $GITHUB_ENV - name: Push multiarch manifest run: | diff --git a/Makefile b/Makefile index a1119fa5..a243c3b4 100644 --- a/Makefile +++ b/Makefile @@ -129,20 +129,3 @@ tidy: check-scripts: # Fail if any of these files have warnings scripts/shellcheck.sh - -## check-version: run check to ensure version in CHANGELOG.md matches references in files -.PHONY: check-version -check-version: -# Fail if syncing version would produce changes - scripts/version-sync.sh -c \ - -s CHANGELOG.md \ - -f ${PACKAGE_NAME}/api/app.py release \ - -f ${PACKAGE_NAME}/api/general.py release - -## version-sync: update references to version with most recent version from CHANGELOG.md -.PHONY: version-sync -version-sync: - scripts/version-sync.sh \ - -s CHANGELOG.md \ - -f ${PACKAGE_NAME}/api/app.py release \ - -f ${PACKAGE_NAME}/api/general.py release diff --git a/scripts/version-sync.sh b/scripts/version-sync.sh deleted file mode 100755 index 4a62d26e..00000000 --- a/scripts/version-sync.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash -function usage { - echo "Usage: $(basename "$0") [-c] -f FILE_TO_CHANGE REPLACEMENT_FORMAT [-f FILE_TO_CHANGE REPLACEMENT_FORMAT ...]" 2>&1 - echo 'Synchronize files to latest version in source file' - echo ' -s Specifies source file for version (default is CHANGELOG.md)' - echo ' -f Specifies a file to change and the format for searching and replacing versions' - echo ' FILE_TO_CHANGE is the file to be updated/checked for updates' - echo ' REPLACEMENT_FORMAT is one of (semver, release, api-release)' - echo ' semver indicates to look for a full semver version and replace with the latest full version' - echo ' release indicates to look for a release semver version (x.x.x) and replace with the latest release version' - echo ' api-release indicates to look for a release semver version in the context of an api route and replace with the latest release version' - echo ' -c Compare versions and output proposed changes without changing anything.' -} - -function getopts-extra () { - declare -i i=1 - # if the next argument is not an option, then append it to array OPTARG - while [[ ${OPTIND} -le $# && ${!OPTIND:0:1} != '-' ]]; do - OPTARG[i]=${!OPTIND} - ((i += 1)) - ((OPTIND += 1)) - done -} - -# Parse input options -declare CHECK=0 -declare SOURCE_FILE="CHANGELOG.md" -declare -a FILES_TO_CHECK=() -declare -a REPLACEMENT_FORMATS=() -declare args -declare OPTIND OPTARG opt -while getopts ":hcs:f:" opt; do - case $opt in - h) - usage - exit 0 - ;; - c) - CHECK=1 - ;; - s) - SOURCE_FILE="$OPTARG" - ;; - f) - getopts-extra "$@" - args=( "${OPTARG[@]}" ) - # validate length of args, should be 2 - if [ ${#args[@]} -eq 2 ]; then - FILES_TO_CHECK+=( "${args[0]}" ) - REPLACEMENT_FORMATS+=( "${args[1]}" ) - else - echo "Exactly 2 arguments must follow -f option." >&2 - exit 1 - fi - ;; - \?) - echo "Invalid option: -$OPTARG." >&2 - usage - exit 1 - ;; - esac -done - -# Parse REPLACEMENT_FORMATS -RE_SEMVER_FULL="(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?" -RE_RELEASE="(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" -RE_API_RELEASE="v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" -# Pull out semver appearing earliest in SOURCE_FILE. -LAST_VERSION=$(grep -o -m 1 -E "${RE_SEMVER_FULL}" "$SOURCE_FILE") -LAST_RELEASE=$(grep -o -m 1 -E "${RE_RELEASE}($|[^-+])" "$SOURCE_FILE" | grep -o -m 1 -E "${RE_RELEASE}") -LAST_API_RELEASE="v$(grep -o -m 1 -E "${RE_RELEASE}($|[^-+])$" "$SOURCE_FILE" | grep -o -m 1 -E "${RE_RELEASE}")" -declare -a RE_SEMVERS=() -declare -a UPDATED_VERSIONS=() -for i in "${!REPLACEMENT_FORMATS[@]}"; do - REPLACEMENT_FORMAT=${REPLACEMENT_FORMATS[$i]} - case $REPLACEMENT_FORMAT in - semver) - RE_SEMVERS+=( "$RE_SEMVER_FULL" ) - UPDATED_VERSIONS+=( "$LAST_VERSION" ) - ;; - release) - RE_SEMVERS+=( "$RE_RELEASE" ) - UPDATED_VERSIONS+=( "$LAST_RELEASE" ) - ;; - api-release) - RE_SEMVERS+=( "$RE_API_RELEASE" ) - UPDATED_VERSIONS+=( "$LAST_API_RELEASE" ) - ;; - *) - echo "Invalid replacement format: \"${REPLACEMENT_FORMAT}\". Use semver, release, or api-release" >&2 - exit 1 - ;; - esac -done - -if [ -z "$LAST_VERSION" ]; -then - # No match to semver regex in SOURCE_FILE, so no version to go from. - printf "Error: Unable to find latest version from %s.\n" "$SOURCE_FILE" - exit 1 -fi - -# Search files in FILES_TO_CHECK and change (or get diffs) -declare FAILED_CHECK=0 - -for i in "${!FILES_TO_CHECK[@]}"; do - FILE_TO_CHANGE=${FILES_TO_CHECK[$i]} - RE_SEMVER=${RE_SEMVERS[$i]} - UPDATED_VERSION=${UPDATED_VERSIONS[$i]} - FILE_VERSION=$(grep -o -m 1 -E "${RE_SEMVER}" "$FILE_TO_CHANGE") - if [ -z "$FILE_VERSION" ]; - then - # No match to semver regex in VERSIONFILE, so nothing to replace - printf "Error: No semver version found in file %s.\n" "$FILE_TO_CHANGE" - exit 1 - else - # Replace semver in VERSIONFILE with semver obtained from SOURCE_FILE - TMPFILE=$(mktemp /tmp/new_version.XXXXXX) - # Check sed version, exit if version < 4.3 - if ! sed --version > /dev/null 2>&1; then - CURRENT_VERSION=1.archaic - else - CURRENT_VERSION=$(sed --version | head -n1 | cut -d" " -f4) - fi - REQUIRED_VERSION="4.3" - if [ "$(printf '%s\n' "$REQUIRED_VERSION" "$CURRENT_VERSION" | sort -V | head -n1)" != "$REQUIRED_VERSION" ]; then - echo "sed version must be >= ${REQUIRED_VERSION}" && exit 1 - fi - sed -E -r "s/$RE_SEMVER/$UPDATED_VERSION/" "$FILE_TO_CHANGE" > "$TMPFILE" - if [ $CHECK == 1 ]; - then - DIFF=$(diff "$FILE_TO_CHANGE" "$TMPFILE" ) - if [ -z "$DIFF" ]; - then - printf "version sync would make no changes to %s.\n" "$FILE_TO_CHANGE" - rm "$TMPFILE" - else - FAILED_CHECK=1 - printf "version sync would make the following changes to %s:\n%s\n" "$FILE_TO_CHANGE" "$DIFF" - rm "$TMPFILE" - fi - else - cp "$TMPFILE" "$FILE_TO_CHANGE" - rm "$TMPFILE" - fi - fi -done - -# Exit with code determined by whether changes were needed in a check. -if [ ${FAILED_CHECK} -ne 0 ]; then - exit 1 -else - exit 0 -fi From e33180004adf875b7c679812c1a473b31c41585c Mon Sep 17 00:00:00 2001 From: Emily Voss Date: Thu, 1 May 2025 11:01:31 -0700 Subject: [PATCH 3/3] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6a8310f..bab94c56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.0.85 +* Set version in '__version__.py' file +* Remove version-sync.py script and preprocessing_family_pipeline.yaml file + ## 0.0.84 * Patch h11 CVE * bump httpcore version due to h11 dependency