From 76db8927c665e75d9402025280531f0ee95ca30b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 30 Apr 2025 14:30:44 -0500 Subject: [PATCH 1/7] INTPYTHON-615 Set up GitHub workflows for automated releases --- .github/workflows/release-python.yml | 154 +++++++++++++++------------ .github/workflows/test-python.yml | 16 +-- .github/workflows/zizmor.yml | 4 +- 3 files changed, 97 insertions(+), 77 deletions(-) diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index aa898ee..35de6d4 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -1,12 +1,25 @@ -name: Python Wheels +name: Release on: - push: - branches: ["main"] - tags: - - "**" - pull_request: workflow_dispatch: + inputs: + following_version: + description: "The post (dev) version to set" + dry_run: + description: "Dry Run?" + default: false + type: boolean + schedule: + - cron: '30 5 * * *' + +env: + # Changes per repo + PRODUCT_NAME: python-bsonjs + # Constant + # inputs will be empty on a scheduled run. so, we only set dry_run + # to 'false' when the input is set to 'false'. + DRY_RUN: ${{ ! contains(inputs.dry_run, 'false') }} + FOLLOWING_VERSION: ${{ inputs.following_version || '' }} concurrency: group: wheels-${{ github.ref }} @@ -17,83 +30,90 @@ defaults: shell: bash -eux {0} jobs: - - build_dist: - name: Build Distribution Files - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: actions/setup-python@v5 - with: - # Build sdist on lowest supported Python - python-version: '3.10' - - - name: Install build - run: | - python -m pip install build - - - name: build the dist files - run: | - python -m build . - - - name: Upload the dist files - uses: actions/upload-artifact@v4 - with: - name: dist-${{ github.run_id }} - path: ./dist/*.* - - test_dist: - needs: [build_dist] - name: Test Distribution Files + pre-publish: + environment: release runs-on: ubuntu-latest + if: github.repository_owner == 'mongodb-labs' || github.event_name == 'workflow_dispatch' + permissions: + id-token: write + contents: write + outputs: + version: ${{ steps.pre-publish.outputs.version }} steps: - - uses: actions/checkout@v4 + - uses: mongodb-labs/drivers-github-tools/secure-checkout@v2 with: - fetch-depth: 0 - persist-credentials: false - - - uses: actions/setup-python@v5 + app_id: ${{ vars.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: mongodb-labs/drivers-github-tools/setup@v2 with: - # Build sdist on lowest supported Python - python-version: '3.10' - - - name: Download the dists - uses: actions/download-artifact@v4 + aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} + aws_region_name: ${{ vars.AWS_REGION_NAME }} + aws_secret_id: ${{ secrets.AWS_SECRET_ID }} + artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }} + - uses: mongodb-labs/drivers-github-tools/python-labs/pre-publish@v2 + id: pre-publish with: - name: dist-${{ github.run_id }} - path: dist/ + dry_run: ${{ env.DRY_RUN }} - - name: Test the sdist - run: | - cd dist - pip install *.tar.gz - python -c "import pymongo_voyageai" - pip uninstall -y pymongo_voyageai + build-dist: + needs: [pre-publish] + uses: ./.github/workflows/dist.yml + with: + ref: ${{ needs.pre-publish.outputs.version }} - - name: Test the wheel - run: | - cd dist - pip install *.whl - python -c "import pymongo_voyageai" - pip uninstall -y pymongo_voyageai + static-scan: + needs: [pre-publish] + uses: ./.github/workflows/codeql.yml + with: + ref: ${{ needs.pre-publish.outputs.version }} publish: # https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#publishing-the-distribution-to-pypi - needs: [test_dist] - if: startsWith(github.ref, 'refs/tags/') + needs: [build-dist, static-scan] + if: (github.repository_owner == 'mongodb-labs' && github.event_name != 'pull_request') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest environment: release permissions: id-token: write steps: - - name: Download the dists + - name: Download all the dists uses: actions/download-artifact@v4 with: - name: dist-${{ github.run_id }} + name: all-dist-${{ github.run_id }} path: dist/ + - name: Publish package distributions to TestPyPI + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + skip-existing: true + attestations: ${{ env.DRY_RUN }} - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + if: startsWith(env.DRY_RUN, 'false') + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # release/v1 + + post-publish: + needs: [publish] + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + contents: write + attestations: write + security-events: write + steps: + - uses: mongodb-labs/drivers-github-tools/secure-checkout@v2 + with: + app_id: ${{ vars.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: mongodb-labs/drivers-github-tools/setup@v2 + with: + aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} + aws_region_name: ${{ vars.AWS_REGION_NAME }} + aws_secret_id: ${{ secrets.AWS_SECRET_ID }} + artifactory_username: ${{ vars.ARTIFACTORY_USERNAME }} + - uses: mongodb-labs/drivers-github-tools/python-labs/post-publish@v2 + with: + following_version: ${{ env.FOLLOWING_VERSION }} + product_name: ${{ env.PRODUCT_NAME }} + token: ${{ github.token }} + dry_run: ${{ env.DRY_RUN }} diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index d455b57..6c85c25 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -25,11 +25,11 @@ jobs: persist-credentials: false fetch-depth: 0 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 with: enable-cache: true python-version: ${{ matrix.python-version }} - - uses: extractions/setup-just@v3 + - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - run: just install - run: just lint - run: just docs @@ -47,11 +47,11 @@ jobs: persist-credentials: false fetch-depth: 0 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 with: enable-cache: true python-version: ${{ matrix.python-version }} - - uses: extractions/setup-just@v3 + - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - uses: mongodb-labs/drivers-evergreen-tools@master with: local-atlas: '1' @@ -66,17 +66,17 @@ jobs: persist-credentials: false fetch-depth: 0 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 with: enable-cache: true python-version: ${{ env.MIN_PYTHON }} - - uses: extractions/setup-just@v3 + - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 with: enable-cache: true python-version: ${{ env.MIN_PYTHON }} - - uses: extractions/setup-just@v3 + - uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - uses: mongodb-labs/drivers-evergreen-tools@master with: local-atlas: '1' diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 0fbdbd6..5b55068 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -18,7 +18,7 @@ jobs: with: persist-credentials: false - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 + uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1 - name: Get zizmor run: cargo install zizmor - name: Run zizmor @@ -26,7 +26,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3 with: sarif_file: results.sarif category: zizmor From 3b1177b74b49043fa48da896afe445cd0d435032 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Wed, 30 Apr 2025 14:30:53 -0500 Subject: [PATCH 2/7] add files --- .github/workflows/codeql.yml | 77 ++++++++++++++++++++++++++++++++++++ .github/workflows/dist.yml | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/dist.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..95270d0 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,77 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "master", "*" ] + pull_request: + branches: [ "master", "*" ] + schedule: + - cron: '35 23 * * 5' + workflow_call: + inputs: + ref: + required: true + type: string + +jobs: + analyze: + name: Analyze ${{ matrix.language }} + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + # required for all workflows + security-events: write + # required to fetch internal or private CodeQL packs + packages: read + actions: read + contents: read + strategy: + fail-fast: false + matrix: + include: + - language: python + - language: actions + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@28deaeda66b76a05916b6923827895f2b14ab387 # v3 + with: + languages: ${{ matrix.language }} + build-mode: none + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + queries: security-extended + config: | + paths-ignore: + - 'test/**' + + - shell: bash + if: matrix.language == 'python' + run: | + pip install -e . + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@28deaeda66b76a05916b6923827895f2b14ab387 # v3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml new file mode 100644 index 0000000..a544423 --- /dev/null +++ b/.github/workflows/dist.yml @@ -0,0 +1,74 @@ +name: Python Dist + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+[a-b][0-9]+" + - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" + workflow_dispatch: + pull_request: + workflow_call: + inputs: + ref: + required: true + type: string + +concurrency: + group: dist-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -eux {0} + +jobs: + make_dist: + name: Make Dist + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + # Build sdist on lowest supported Python + python-version: '3.9' + + - name: Install python requirements + run: | + python -m pip install uv rust-just build + + - name: Build Dist + run: | + python -m build . + + - name: Test SDist + run: | + python -m pip install dist/*.gz + cd .. + python -c "from pymongo_voyageai import PyMongoVoyageAI" + + - uses: actions/upload-artifact@v4 + with: + name: "dist" + path: ./dist/*.* + + collect_dist: + runs-on: ubuntu-latest + needs: [make_dist] + name: Download Dist + steps: + - name: Download all workflow run artifacts + uses: actions/download-artifact@v4 + - name: Flatten directory + working-directory: . + run: | + find . -mindepth 2 -type f -exec mv {} . \; + find . -type d -empty -delete + - uses: actions/upload-artifact@v4 + with: + name: all-dist-${{ github.run_id }} + path: "./*" From abf05b3e0a8cd3d53d5f3a11fd187350d9a39832 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 1 May 2025 13:43:41 -0500 Subject: [PATCH 3/7] cleanup and add gitignore --- .github/workflows/codeql.yml | 2 +- .github/workflows/dist.yml | 2 +- .gitignore | 62 ++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 95270d0..462cdd2 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -52,7 +52,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.x + python-version: "3.10" # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index a544423..f41e940 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/setup-python@v5 with: # Build sdist on lowest supported Python - python-version: '3.9' + python-version: '3.10' - name: Install python requirements run: | diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6d10eec --- /dev/null +++ b/.gitignore @@ -0,0 +1,62 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Sphinx documentation +docs/_build/ + +# Environments +.env +.venv + +# ruff +.ruff_cache/ + +# End of https://www.toptal.com/developers/gitignore/api/python From 0ae769f60db99a3a016b1ccac9df64b27dc2ddd2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 1 May 2025 14:04:01 -0500 Subject: [PATCH 4/7] address automated review --- .github/workflows/dist.yml | 4 ++++ .github/workflows/release-python.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index f41e940..4aece61 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -15,6 +15,10 @@ on: required: true type: string +permissions: + contents: read + actions: read + concurrency: group: dist-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml index 35de6d4..4bef986 100644 --- a/.github/workflows/release-python.yml +++ b/.github/workflows/release-python.yml @@ -58,12 +58,16 @@ jobs: build-dist: needs: [pre-publish] uses: ./.github/workflows/dist.yml + permissions: + contents: read with: ref: ${{ needs.pre-publish.outputs.version }} static-scan: needs: [pre-publish] uses: ./.github/workflows/codeql.yml + permissions: + contents: read with: ref: ${{ needs.pre-publish.outputs.version }} From 464808aa48811be5a8c23717118ac031e1ba0794 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 1 May 2025 14:08:36 -0500 Subject: [PATCH 5/7] add zizmor config --- .github/zizmor.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/zizmor.yml diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 0000000..305efdf --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,7 @@ +rules: + unpinned-uses: + config: + policies: + actions/*: ref-pin + mongodb-labs/drivers-github-tools: ref-pin + "*": hash-pin From 756f584936f8b0e59d194d7b5668972ea58ae8dd Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 1 May 2025 14:16:12 -0500 Subject: [PATCH 6/7] address automated review --- .github/workflows/test-python.yml | 6 ++++++ .github/zizmor.yml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 6c85c25..69e6a00 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -19,6 +19,8 @@ env: jobs: static: runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v4 with: @@ -36,6 +38,8 @@ jobs: - run: just doctest build: runs-on: ubuntu-latest + permissions: + contents: read strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] @@ -60,6 +64,8 @@ jobs: build-min: runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@v4 with: diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 305efdf..10fd4cd 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -3,5 +3,5 @@ rules: config: policies: actions/*: ref-pin - mongodb-labs/drivers-github-tools: ref-pin - "*": hash-pin + mongodb-labs/drivers-github-tools/*: ref-pin + mongodb-labs/drivers-evergreen-tools: ref-pin From d948a3f4749e4b9ad1ca8e64e3553e2aae2aedcb Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 2 May 2025 11:05:34 -0500 Subject: [PATCH 7/7] bump to alpha version --- pymongo_voyageai/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymongo_voyageai/_version.py b/pymongo_voyageai/_version.py index 4b4a921..8f02035 100644 --- a/pymongo_voyageai/_version.py +++ b/pymongo_voyageai/_version.py @@ -1 +1 @@ -__version__ = "0.1.0.dev0" +__version__ = "0.1.0a0"