Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update the manylinux wheel glibc version to 2.28 #21650

Merged
merged 2 commits into from
Nov 16, 2024
Merged

update the manylinux wheel glibc version to 2.28 #21650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e3bd6b
manylinux wip
cburroughs Nov 15, 2024
7bfdc59
lint and docs
cburroughs Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 23 additions & 7 deletions .github/workflows/release.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
jobs:
build_wheels_linux_arm64:
container:
image: quay.io/pypa/manylinux2014_aarch64:latest
image: quay.io/pypa/manylinux_2_28_aarch64:latest
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
PANTS_REMOTE_CACHE_READ: 'false'
Expand All @@ -23,7 +23,7 @@ jobs:
- run-id=${{ github.run_id }}
steps:
- name: Check out code
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 10
ref: ${{ needs.release_info.outputs.build-ref }}
Expand All @@ -43,6 +43,14 @@ jobs:

echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH

echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH

echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH

echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH

echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH

'
- name: Install Protoc
uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9
Expand All @@ -58,7 +66,7 @@ jobs:
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: logs-wheels-and-pex-Linux-ARM64
overwrite: 'true'
Expand All @@ -83,7 +91,7 @@ jobs:
timeout-minutes: 90
build_wheels_linux_x86_64:
container:
image: quay.io/pypa/manylinux2014_x86_64:latest
image: quay.io/pypa/manylinux_2_28_x86_64:latest
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
PANTS_REMOTE_CACHE_READ: 'false'
Expand All @@ -96,7 +104,7 @@ jobs:
- ubuntu-22.04
steps:
- name: Check out code
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 10
ref: ${{ needs.release_info.outputs.build-ref }}
Expand All @@ -116,14 +124,22 @@ jobs:

echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH

echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH

echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH

echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH

echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH

'
- name: Install Protoc
uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
version: 23.x
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: 1.19.5
- env:
Expand All @@ -137,7 +153,7 @@ jobs:
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: logs-wheels-and-pex-Linux-x86_64
overwrite: 'true'
Expand Down
46 changes: 31 additions & 15 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,14 +325,14 @@ jobs:
timeout-minutes: 60
build_wheels_linux_arm64:
container:
image: quay.io/pypa/manylinux2014_aarch64:latest
image: quay.io/pypa/manylinux_2_28_aarch64:latest
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
MODE: debug
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true')) && (needs.classify_changes.outputs.docs_only
!= 'true')
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config
== 'true')) && (needs.classify_changes.outputs.docs_only != 'true')
name: Build wheels (Linux-ARM64)
needs:
- classify_changes
Expand All @@ -344,7 +344,7 @@ jobs:
- run-id=${{ github.run_id }}
steps:
- name: Check out code
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 10
- name: Configure Git
Expand All @@ -362,6 +362,14 @@ jobs:

echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH

echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH

echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH

echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH

echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH

'
- name: Install Protoc
uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9
Expand All @@ -377,30 +385,30 @@ jobs:
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: logs-wheels-and-pex-Linux-ARM64
overwrite: 'true'
path: .pants.d/workdir/*.log
timeout-minutes: 90
build_wheels_linux_x86_64:
container:
image: quay.io/pypa/manylinux2014_x86_64:latest
image: quay.io/pypa/manylinux_2_28_x86_64:latest
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
MODE: debug
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true')) && (needs.classify_changes.outputs.docs_only
!= 'true')
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config
== 'true')) && (needs.classify_changes.outputs.docs_only != 'true')
name: Build wheels (Linux-x86_64)
needs:
- classify_changes
runs-on:
- ubuntu-22.04
steps:
- name: Check out code
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 10
- name: Configure Git
Expand All @@ -418,14 +426,22 @@ jobs:

echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH

echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH

echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH

echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH

echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH

'
- name: Install Protoc
uses: arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
version: 23.x
- name: Install Go
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: 1.19.5
- env:
Expand All @@ -439,7 +455,7 @@ jobs:
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: logs-wheels-and-pex-Linux-x86_64
overwrite: 'true'
Expand All @@ -451,8 +467,8 @@ jobs:
MODE: debug
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true')) && (needs.classify_changes.outputs.docs_only
!= 'true')
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config
== 'true')) && (needs.classify_changes.outputs.docs_only != 'true')
name: Build wheels (macOS10-15-x86_64)
needs:
- classify_changes
Expand Down Expand Up @@ -520,8 +536,8 @@ jobs:
MODE: debug
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true')) && (needs.classify_changes.outputs.docs_only
!= 'true')
if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config
== 'true')) && (needs.classify_changes.outputs.docs_only != 'true')
name: Build wheels (macOS11-ARM64)
needs:
- classify_changes
Expand Down
2 changes: 2 additions & 0 deletions docs/notes/2.25.x.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ The "legacy" options system is removed in this release. All options parsing is n

### Plugin API changes

The oldest [glibc version](https://www.sourceware.org/glibc/wiki/Glibc%20Timeline) supported by the published Pants wheels is now 2.28. This should have no effect unless you are running on extremely old Linux distributions. See <https://github.com/pypa/manylinux> for background context on Python wheels and C libraries.

## Full Changelog

For the full changelog, see the individual GitHub Releases for this series: <https://github.com/pantsbuild/pants/releases>
75 changes: 31 additions & 44 deletions src/python/pants_release/generate_github_workflows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,40 +18,27 @@
from pants_release.common import die


def action(name: str, node16_compat: bool = False) -> str:
# Versions of actions compatible with node16 and the `ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION` setting.
# glibc 2.17 is required to build manylinux_2014 wheels, but node.js does do not ship glibc 2.17 compatible
# binaries for node >= v17.
if node16_compat:
version_map = {
"checkout": "actions/checkout@v3",
"upload-artifact": "actions/upload-artifact@v3",
"setup-go": "actions/setup-go@v4",
}
else:
version_map = {
"action-send-mail": "dawidd6/[email protected]",
"cache": "actions/cache@v4",
"checkout": "actions/checkout@v4",
"download-artifact": "actions/download-artifact@v4",
"github-action-required-labels": "mheap/[email protected]",
"rust-cache": "benjyw/rust-cache@5ed697a6894712d2854c80635bb00a2496ea307a",
"setup-go": "actions/setup-go@v5",
"setup-java": "actions/setup-java@v4",
"setup-node": "actions/setup-node@v4",
"setup-protoc": "arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9",
"setup-python": "actions/setup-python@v5",
"slack-github-action": "slackapi/[email protected]",
"upload-artifact": "actions/upload-artifact@v4",
}
def action(name: str) -> str:
version_map = {
"action-send-mail": "dawidd6/[email protected]",
"cache": "actions/cache@v4",
"checkout": "actions/checkout@v4",
"download-artifact": "actions/download-artifact@v4",
"github-action-required-labels": "mheap/[email protected]",
"rust-cache": "benjyw/rust-cache@5ed697a6894712d2854c80635bb00a2496ea307a",
"setup-go": "actions/setup-go@v5",
"setup-java": "actions/setup-java@v4",
"setup-node": "actions/setup-node@v4",
"setup-protoc": "arduino/setup-protoc@9b1ee5b22b0a3f1feb8c2ff99b32c89b3c3191e9",
"setup-python": "actions/setup-python@v5",
"slack-github-action": "slackapi/[email protected]",
"upload-artifact": "actions/upload-artifact@v4",
}
try:
return version_map[name]
except KeyError as e:
configured_version = (
"Node 16 compatible version configured" if node16_compat else "version configured"
)
raise ValueError(
f"Requested github action '{name}' does not have a {configured_version}.\n"
f"Requested github action '{name}' does not have a version configured.\n"
f"Current known versions: {version_map}"
) from e

Expand Down Expand Up @@ -126,7 +113,7 @@ def hash_files(path: str) -> str:
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.12", "3.13", "3.11"]

DONT_SKIP_RUST = "needs.classify_changes.outputs.rust == 'true'"
DONT_SKIP_WHEELS = "needs.classify_changes.outputs.release == 'true'"
DONT_SKIP_WHEELS = "needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true'"
IS_PANTS_OWNER = "github.repository_owner == 'pantsbuild'"

# NB: This overrides `pants.ci.toml`.
Expand Down Expand Up @@ -238,7 +225,7 @@ def checkout(
# We need to fetch a few commits back, to be able to access HEAD^2 in the PR case.
{
"name": "Check out code",
"uses": action("checkout", node16_compat=containerized),
"uses": action("checkout"),
"with": {
**fetch_depth_opt,
**({"ref": ref} if ref else {}),
Expand Down Expand Up @@ -367,10 +354,10 @@ def install_jdk() -> Step:
}


def install_go(node16_compat: bool = False) -> Step:
def install_go() -> Step:
return {
"name": "Install Go",
"uses": action("setup-go", node16_compat=node16_compat),
"uses": action("setup-go"),
"with": {"go-version": "1.19.5"},
}

Expand Down Expand Up @@ -582,10 +569,10 @@ def bootstrap_pants(self) -> Sequence[Step]:
self.native_binaries_upload(),
]

def upload_log_artifacts(self, name: str, node16_compat: bool = False) -> Step:
def upload_log_artifacts(self, name: str) -> Step:
return {
"name": "Upload pants.log",
"uses": action("upload-artifact", node16_compat=node16_compat),
"uses": action("upload-artifact"),
"if": "always()",
"continue-on-error": True,
"with": {
Expand Down Expand Up @@ -834,9 +821,9 @@ def build_wheels_job(
# the code, install rustup and expose Pythons.
# TODO: Apply rust caching here.
if platform == Platform.LINUX_X86_64:
container = {"image": "quay.io/pypa/manylinux2014_x86_64:latest"}
container = {"image": "quay.io/pypa/manylinux_2_28_x86_64:latest"}
elif platform == Platform.LINUX_ARM64:
container = {"image": "quay.io/pypa/manylinux2014_aarch64:latest"}
container = {"image": "quay.io/pypa/manylinux_2_28_aarch64:latest"}
else:
container = None

Expand All @@ -851,6 +838,10 @@ def build_wheels_job(
echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH
echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH
echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH
echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH
echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH
echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH
echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH
"""
),
},
Expand Down Expand Up @@ -889,11 +880,7 @@ def build_wheels_job(
"steps": [
*initial_steps,
install_protoc(), # for prost crate
*(
[]
if platform == Platform.LINUX_ARM64
else [install_go(node16_compat=bool(container))]
),
*([] if platform == Platform.LINUX_ARM64 else [install_go()]),
{
"name": "Build wheels",
"run": "./pants run src/python/pants_release/release.py -- build-wheels",
Expand All @@ -904,7 +891,7 @@ def build_wheels_job(
"run": "./pants package src/python/pants:pants-pex",
"env": helper.platform_env(),
},
helper.upload_log_artifacts(name="wheels-and-pex", node16_compat=bool(container)),
helper.upload_log_artifacts(name="wheels-and-pex"),
*(
[
{
Expand Down
Loading