From ed94aebb1102b0d7c641fa97bd7ee975897f9d84 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 18:45:59 -0700 Subject: [PATCH 1/7] Replace various *proto packages with xorgproto As part of trying to stand up native Linux ARM builds, I ran into a failure with xproto's config.guess not recognizing ARM. That package not releasing since 2016 clued me in that it was likely not a required part of X11 any more. I looked at Debian's dependency tree and there was no xproto in sight. But there was an xorgproto. And sure enough if you install xorgproto instead of xproto in the build environment, things "just work." Clued into the potential for outdated packages, I looked at other proto packages. I pulled the thread and inputproto, kbproto, and xextproto all appeared to be dead and replaced by xorgproto. So this commit deletes inputproto, kbproto, xextproto, and xproto and effectively replaces their use with xorgproto. Good riddance. --- cpython-unix/Makefile | 20 ++---------------- cpython-unix/build-inputproto.sh | 24 ---------------------- cpython-unix/build-kbproto.sh | 24 ---------------------- cpython-unix/build-xextproto.sh | 35 -------------------------------- cpython-unix/build-xproto.sh | 35 -------------------------------- cpython-unix/build.py | 12 ++--------- pythonbuild/downloads.py | 24 ---------------------- 7 files changed, 4 insertions(+), 170 deletions(-) delete mode 100755 cpython-unix/build-inputproto.sh delete mode 100755 cpython-unix/build-kbproto.sh delete mode 100755 cpython-unix/build-xextproto.sh delete mode 100755 cpython-unix/build-xproto.sh diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 0e859784..196b6bbf 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -117,12 +117,6 @@ $(OUTDIR)/bzip2-$(BZIP2_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $( $(OUTDIR)/expat-$(EXPAT_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-expat.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) expat -$(OUTDIR)/inputproto-$(INPUTPROTO_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-inputproto.sh - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) inputproto - -$(OUTDIR)/kbproto-$(KBPROTO_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-kbproto.sh - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) kbproto - $(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libffi-3.3.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) libffi-3.3 @@ -135,10 +129,6 @@ $(OUTDIR)/libpthread-stubs-$(LIBPTHREAD_STUBS_VERSION)-$(PACKAGE_SUFFIX).tar: $( LIBX11_DEPENDS = \ $(PYTHON_DEP_DEPENDS) \ $(HERE)/build-libX11.sh \ - $(OUTDIR)/xproto-$(XPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(OUTDIR)/xextproto-$(XEXTPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(OUTDIR)/kbproto-$(KBPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(OUTDIR)/inputproto-$(INPUTPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/libxcb-$(LIBXCB_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/xtrans-$(XTRANS_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/xorgproto-$(XORGPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ @@ -151,7 +141,7 @@ LIBXAU_DEPENDS = \ $(PYTHON_DEP_DEPENDS) \ $(HERE)/build-libXau.sh \ $(OUTDIR)/x11-util-macros-$(X11_UTIL_MACROS_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(OUTDIR)/xproto-$(XPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ + $(OUTDIR)/xorgproto-$(XORGPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ $(NULL) $(OUTDIR)/libXau-$(LIBXAU_VERSION)-$(PACKAGE_SUFFIX).tar: $(LIBXAU_DEPENDS) @@ -163,7 +153,7 @@ LIBXCB_DEPENDS = \ $(OUTDIR)/image-$(DOCKER_IMAGE_XCB).tar \ $(OUTDIR)/xcb-proto-$(XCB_PROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/libXau-$(LIBXAU_VERSION)-$(PACKAGE_SUFFIX).tar \ - $(OUTDIR)/xproto-$(XPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ + $(OUTDIR)/xorgproto-$(XORGPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/libpthread-stubs-$(LIBPTHREAD_STUBS_VERSION)-$(PACKAGE_SUFFIX).tar \ $(NULL) @@ -231,15 +221,9 @@ $(OUTDIR)/x11-util-macros-$(X11_UTIL_MACROS_VERSION)-$(PACKAGE_SUFFIX).tar: $(PY $(OUTDIR)/xcb-proto-$(XCB_PROTO_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-xcb-proto.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_XCB) xcb-proto -$(OUTDIR)/xextproto-$(XEXTPROTO_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-xextproto.sh - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) xextproto - $(OUTDIR)/xorgproto-$(XORGPROTO_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-xorgproto.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) xorgproto -$(OUTDIR)/xproto-$(XPROTO_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-xproto.sh - $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) xproto - $(OUTDIR)/xtrans-$(XTRANS_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-xtrans.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) xtrans diff --git a/cpython-unix/build-inputproto.sh b/cpython-unix/build-inputproto.sh deleted file mode 100755 index 9ab52363..00000000 --- a/cpython-unix/build-inputproto.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -set -ex - -ROOT=`pwd` - -pkg-config --version - -export PATH=/tools/${TOOLCHAIN}/bin:/tools/host/bin:$PATH -export PKG_CONFIG_PATH=/tools/deps/share/pkgconfig - -tar -xf inputproto-${INPUTPROTO_VERSION}.tar.gz -pushd inputproto-${INPUTPROTO_VERSION} - -CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ - --build=${BUILD_TRIPLE} \ - --host=${TARGET_TRIPLE} \ - --prefix=/tools/deps - -make -j `nproc` -make -j `nproc` install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build-kbproto.sh b/cpython-unix/build-kbproto.sh deleted file mode 100755 index 031493e8..00000000 --- a/cpython-unix/build-kbproto.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -set -ex - -ROOT=`pwd` - -pkg-config --version - -export PATH=/tools/${TOOLCHAIN}/bin:/tools/host/bin:$PATH -export PKG_CONFIG_PATH=/tools/deps/share/pkgconfig - -tar -xf kbproto-${KBPROTO_VERSION}.tar.gz -pushd kbproto-${KBPROTO_VERSION} - -CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ - --build=${BUILD_TRIPLE} \ - --host=${TARGET_TRIPLE} \ - --prefix=/tools/deps - -make -j `nproc` -make -j `nproc` install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build-xextproto.sh b/cpython-unix/build-xextproto.sh deleted file mode 100755 index 090e3f0e..00000000 --- a/cpython-unix/build-xextproto.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -set -ex - -ROOT=`pwd` - -pkg-config --version - -export PATH=/tools/${TOOLCHAIN}/bin:/tools/host/bin:$PATH -export PKG_CONFIG_PATH=/tools/deps/share/pkgconfig - -tar -xf xextproto-${XEXTPROTO_VERSION}.tar.gz -pushd xextproto-${XEXTPROTO_VERSION} - -EXTRA_CONFIGURE_FLAGS= -if [ -n "${CROSS_COMPILING}" ]; then - if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then - # xextproto does not support configuration of musl targets so we pretend the target matches the - # build triple and enable cross-compilation manually - TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')" - EXTRA_CONFIGURE_FLAGS="cross_compiling=yes" - fi -fi - -CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ - --build=${BUILD_TRIPLE} \ - --host=${TARGET_TRIPLE} \ - --prefix=/tools/deps \ - ${EXTRA_CONFIGURE_FLAGS} - -make -j `nproc` -make -j `nproc` install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build-xproto.sh b/cpython-unix/build-xproto.sh deleted file mode 100755 index 6f3d2b6b..00000000 --- a/cpython-unix/build-xproto.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -set -ex - -ROOT=`pwd` - -pkg-config --version - -export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH -export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig - -tar -xf xproto-${XPROTO_VERSION}.tar.gz -pushd xproto-${XPROTO_VERSION} - -EXTRA_CONFIGURE_FLAGS= -if [ -n "${CROSS_COMPILING}" ]; then - if echo "${TARGET_TRIPLE}" | grep -q -- "-unknown-linux-musl"; then - # xproto does not support configuration of musl targets so we pretend the target matches the - # build triple and enable cross-compilation manually - TARGET_TRIPLE="$(echo "${TARGET_TRIPLE}" | sed -e 's/-unknown-linux-musl/-unknown-linux-gnu/g')" - EXTRA_CONFIGURE_FLAGS="cross_compiling=yes" - fi -fi - -CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" ./configure \ - --build=${BUILD_TRIPLE} \ - --host=${TARGET_TRIPLE} \ - --prefix=/tools/deps \ - ${EXTRA_CONFIGURE_FLAGS} - -make -j ${NUM_CPUS} -make -j ${NUM_CPUS} install DESTDIR=${ROOT}/out diff --git a/cpython-unix/build.py b/cpython-unix/build.py index c17c84cd..f058e93a 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -1135,8 +1135,6 @@ def main(): "bdb", "bzip2", "expat", - "inputproto", - "kbproto", "libffi-3.3", "libffi", "libpthread-stubs", @@ -1150,9 +1148,7 @@ def main(): "tcl", "uuid", "x11-util-macros", - "xextproto", "xorgproto", - "xproto", "xtrans", "xz", "zlib", @@ -1182,15 +1178,11 @@ def main(): build_options=build_options, dest_archive=dest_archive, extra_archives={ - "inputproto", - "kbproto", "libpthread-stubs", "libXau", "libxcb", "x11-util-macros", - "xextproto", "xorgproto", - "xproto", "xtrans", }, ) @@ -1205,7 +1197,7 @@ def main(): target_triple=target_triple, build_options=build_options, dest_archive=dest_archive, - extra_archives={"x11-util-macros", "xproto"}, + extra_archives={"x11-util-macros", "xorgproto"}, ) elif action == "xcb-proto": @@ -1230,7 +1222,7 @@ def main(): target_triple=target_triple, build_options=build_options, dest_archive=dest_archive, - extra_archives={"libpthread-stubs", "libXau", "xcb-proto", "xproto"}, + extra_archives={"libpthread-stubs", "libXau", "xcb-proto", "xorgproto"}, ) elif action == "tix": diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 89f3ce08..efd8147e 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -99,24 +99,12 @@ "licenses": ["MIT"], "license_file": "LICENSE.expat.txt", }, - "inputproto": { - "url": "https://www.x.org/archive/individual/proto/inputproto-2.3.2.tar.gz", - "size": 244334, - "sha256": "10eaadd531f38f7c92ab59ef0708ca195caf3164a75c4ed99f0c04f2913f6ef3", - "version": "2.3.2", - }, "jom-windows-bin": { "url": "http://download.qt.io/official_releases/jom/jom_1_1_4.zip", "size": 1696930, "sha256": "d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b", "version": "1.1.4", }, - "kbproto": { - "url": "https://www.x.org/archive/individual/proto/kbproto-1.0.7.tar.gz", - "size": 325858, - "sha256": "828cb275b91268b1a3ea950d5c0c5eb076c678fdf005d517411f89cc8c3bb416", - "version": "1.0.7", - }, "libedit": { "url": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz", "size": 538611, @@ -397,12 +385,6 @@ "sha256": "85cd21e9d9fbc341d0dbf11eace98d55d7db89fda724b0e598855fcddf0944fd", "version": "1.14.1", }, - "xextproto": { - "url": "https://www.x.org/archive/individual/proto/xextproto-7.3.0.tar.gz", - "size": 290814, - "sha256": "1b1bcdf91221e78c6c33738667a57bd9aaa63d5953174ad8ed9929296741c9f5", - "version": "7.3.0", - }, # Newer versions from at least 2023 have build failures for reasons we haven't # fully investigated. "xorgproto": { @@ -411,12 +393,6 @@ "sha256": "38ad1d8316515785d53c5162b4b7022918e03c11d72a5bd9df0a176607f42bca", "version": "2019.1", }, - "xproto": { - "url": "https://www.x.org/archive/individual/proto/xproto-7.0.31.tar.gz", - "size": 367979, - "sha256": "6d755eaae27b45c5cc75529a12855fed5de5969b367ed05003944cf901ed43c7", - "version": "7.0.31", - }, "xtrans": { "url": "https://www.x.org/archive/individual/lib/xtrans-1.5.0.tar.gz", "size": 230197, From dc4c0ae2126e27ce7aa24db7713df1adccd73de7 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 19:13:29 -0700 Subject: [PATCH 2/7] xtrans 1.5.0 -> 1.6.0 This seems to "just work." Let's stay modern. --- pythonbuild/downloads.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index efd8147e..2edccdcf 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -394,10 +394,10 @@ "version": "2019.1", }, "xtrans": { - "url": "https://www.x.org/archive/individual/lib/xtrans-1.5.0.tar.gz", - "size": 230197, - "sha256": "a806f8a92f879dcd0146f3f1153fdffe845f2fc0df9b1a26c19312b7b0a29c86", - "version": "1.5.0", + "url": "https://www.x.org/archive/individual/lib/xtrans-1.6.0.tar.gz", + "size": 239113, + "sha256": "936b74c60b19c317c3f3cb1b114575032528dbdaf428740483200ea874c2ca0a", + "version": "1.6.0", }, # IMPORTANT: xz 5.6 has a backdoor. Be extremely cautious before taking any xz # upgrade since it isn't clear which versions are safe. From baad4feda9aae673384e51d895d310ee5542f6dd Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 19:14:46 -0700 Subject: [PATCH 3/7] x11-util-macros 1.20.1 -> 1.20.2 --- pythonbuild/downloads.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 2edccdcf..755544c1 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -374,10 +374,10 @@ "license_file": "LICENSE.libuuid.txt", }, "x11-util-macros": { - "url": "https://www.x.org/archive/individual/util/util-macros-1.20.1.tar.gz", - "size": 105481, - "sha256": "b373f72887b1394ce2193180a60cb0d1fb8b17bc96ddd770cfd7a808cb489a15", - "version": "1.20.1", + "url": "https://www.x.org/archive/individual/util/util-macros-1.20.2.tar.gz", + "size": 105410, + "sha256": "f642f8964d81acdf06653fdf9dbc210c43ce4bd308bd644a8d573148d0ced76b", + "version": "1.20.2", }, "xcb-proto": { "url": "https://www.x.org/archive/individual/proto/xcb-proto-1.14.1.tar.gz", From 5ed77c972dddc7a1d3f62bc9c42d16e70b8432f4 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 19:18:59 -0700 Subject: [PATCH 4/7] xorgproto 2019.1 -> 2024.1 I think deletion of the legacy proto packages a few commits ago fixed up the build failures the deleted comment alluded to. --- pythonbuild/downloads.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 755544c1..2a0322b6 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -385,13 +385,11 @@ "sha256": "85cd21e9d9fbc341d0dbf11eace98d55d7db89fda724b0e598855fcddf0944fd", "version": "1.14.1", }, - # Newer versions from at least 2023 have build failures for reasons we haven't - # fully investigated. "xorgproto": { - "url": "https://www.x.org/archive/individual/proto/xorgproto-2019.1.tar.gz", - "size": 1119813, - "sha256": "38ad1d8316515785d53c5162b4b7022918e03c11d72a5bd9df0a176607f42bca", - "version": "2019.1", + "url": "https://www.x.org/archive/individual/proto/xorgproto-2024.1.tar.gz", + "size": 1115486, + "sha256": "4f6b9b4faf91e5df8265b71843a91fc73dc895be6210c84117a996545df296ce", + "version": "2024.1", }, "xtrans": { "url": "https://www.x.org/archive/individual/lib/xtrans-1.6.0.tar.gz", From 120252995ecf1138b52d201f4923522e5cf76fba Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sun, 11 May 2025 15:34:07 -0700 Subject: [PATCH 5/7] Update LLVM toolchains to 21.0.4 Let's stay modern. --- pythonbuild/downloads.py | 24 ++++++++++++------------ src/release.rs | 6 +++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 2a0322b6..b777f748 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -175,24 +175,24 @@ }, # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-20-x86_64-linux": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-gnu_only-x86_64-unknown-linux-gnu.tar.zst", - "size": 283261860, - "sha256": "41d3d74e21e064e2e59a4e89feca74d58a5e9e95f73877f3c9ed82ca95607b47", - "version": "20.1.0+20240308", + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-x86_64-unknown-linux-gnu.tar.zst", + "size": 299883811, + "sha256": "32374eb8b32fc79e9022f21eefc848d75fa3c46e68054a5dfc1f68d6f2f20429", + "version": "20.1.4+20250511", }, # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-aarch64-macos": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-aarch64-apple-darwin.tar.zst", - "size": 161006322, - "sha256": "9897bfaab16c930258f614250a1ade1a8f32df027181bd54a61d24b5a04e284c", - "version": "20.1.0+20240308", + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-aarch64-apple-darwin.tar.zst", + "size": 152858186, + "sha256": "d44bf8256b2468339c3b4491edb9c799ab89e466d98d098391286dc86e86a63b", + "version": "20.1.4+20250511", }, # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-x86_64-macos": { - "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-x86_64-apple-darwin.tar.zst", - "size": 160133882, - "sha256": "95cfcb79d752e81735a7a7f7c4cb5a7c16360b6e9a75658a1bd50ce0547d4ad5", - "version": "20.1.0+20240308", + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-x86_64-apple-darwin.tar.zst", + "size": 160140682, + "sha256": "31b35734b678ad22471e31cf6a173c54819a3bca9ffefd4a70d8cdb935d67501", + "version": "20.1.4+20250511", }, "m4": { "url": "https://ftp.gnu.org/gnu/m4/m4-1.4.19.tar.xz", diff --git a/src/release.rs b/src/release.rs index 9d6ec083..b731a241 100644 --- a/src/release.rs +++ b/src/release.rs @@ -569,14 +569,14 @@ pub fn produce_install_only_stripped(tar_gz_path: &Path, llvm_dir: &Path) -> Res static LLVM_URL: Lazy = Lazy::new(|| { if cfg!(target_os = "macos") { if std::env::consts::ARCH == "aarch64" { - Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-aarch64-apple-darwin.tar.zst").unwrap() + Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-aarch64-apple-darwin.tar.zst").unwrap() } else if std::env::consts::ARCH == "x86_64" { - Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-x86_64-apple-darwin.tar.zst").unwrap() + Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-x86_64-apple-darwin.tar.zst").unwrap() } else { panic!("unsupported macOS architecture"); } } else if cfg!(target_os = "linux") { - Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250308/llvm-20.1.0+20250308-gnu_only-x86_64-unknown-linux-gnu.tar.zst").unwrap() + Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-x86_64-unknown-linux-gnu.tar.zst").unwrap() } else { panic!("unsupported platform"); } From 65ffa41103eb0d8626a629c100036248db7e24bc Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 14 May 2025 19:28:27 -0700 Subject: [PATCH 6/7] HACK only build Linux aarch64 in CI --- ci-targets.yaml | 364 +----------------------------------------------- 1 file changed, 1 insertion(+), 363 deletions(-) diff --git a/ci-targets.yaml b/ci-targets.yaml index 70078e87..9a3dac93 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -1,42 +1,5 @@ # Describes the targets that the CI system will build and test on. -darwin: - aarch64-apple-darwin: - arch: aarch64 - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - - x86_64-apple-darwin: - arch: x86_64 - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - linux: aarch64-unknown-linux-gnu: arch: aarch64 @@ -58,329 +21,4 @@ linux: - freethreaded+noopt - freethreaded+lto minimum-python-version: "3.13" - - armv7-unknown-linux-gnueabi: - arch: armv7 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - armv7-unknown-linux-gnueabihf: - arch: armv7 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - s390x-unknown-linux-gnu: - arch: s390x - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - ppc64le-unknown-linux-gnu: - arch: ppc64le - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - riscv64-unknown-linux-gnu: - arch: riscv64 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - - x86_64-unknown-linux-gnu: - arch: x86_64 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64_v2-unknown-linux-gnu: - arch: x86_64 - arch_variant: v2 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64_v3-unknown-linux-gnu: - arch: x86_64 - arch_variant: v3 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64_v4-unknown-linux-gnu: - arch: x86_64 - arch_variant: v4 - libc: gnu - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug - - pgo+lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+pgo+lto - minimum-python-version: "3.13" - run: true - - x86_64-unknown-linux-musl: - arch: x86_64 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - x86_64_v2-unknown-linux-musl: - arch: x86_64 - arch_variant: v2 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - x86_64_v3-unknown-linux-musl: - arch: x86_64 - arch_variant: v3 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - - x86_64_v4-unknown-linux-musl: - arch: x86_64 - arch_variant: v4 - libc: musl - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - debug+static - - noopt+static - - lto+static - - debug - - noopt - - lto - build_options_conditional: - - options: - - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto - minimum-python-version: "3.13" - run: true - -windows: - i686-pc-windows-msvc: - arch: x86 - vcvars: vcvars32.bat - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - pgo - build_options_conditional: - - options: - - freethreaded+pgo - minimum-python-version: "3.13" - - x86_64-pc-windows-msvc: - arch: x86_64 - vcvars: vcvars64.bat - python_versions: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - "3.14" - build_options: - - pgo - build_options_conditional: - - options: - - freethreaded+pgo - minimum-python-version: "3.13" + \ No newline at end of file From ec30c2b2c767e4f9177df6c7ccd4481dedb2c4a0 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sun, 11 May 2025 20:35:02 -0700 Subject: [PATCH 7/7] Enable native builds on Linux aarch64 I just published an LLVM 20 toolchain for aarch64. The toolchain has support for PGO and BOLT. This commit switches the Linux aarch64 builds to be performed natively on aarch64 machines. PGO and BOLT are enabled on the builds, hopefully making them a bit faster. --- .github/workflows/linux.yml | 49 ++++++++++++++-------- ci-runners.yaml | 9 ++-- ci-targets.yaml | 6 +-- cpython-unix/Makefile | 12 +++--- cpython-unix/base.debian9.Dockerfile | 38 +++++++++++++++++ cpython-unix/build-binutils.sh | 8 +++- cpython-unix/build-main.py | 11 ++++- cpython-unix/build.debian9.Dockerfile | 16 +++++++ cpython-unix/build.py | 60 ++++++++++++++++----------- cpython-unix/gcc.debian9.Dockerfile | 14 +++++++ cpython-unix/targets.yml | 21 +++++++--- cpython-unix/xcb.debian9.Dockerfile | 3 ++ docs/running.rst | 2 +- pythonbuild/docker.py | 11 ++--- pythonbuild/downloads.py | 7 ++++ pythonbuild/utils.py | 5 +++ 16 files changed, 199 insertions(+), 73 deletions(-) create mode 100644 cpython-unix/base.debian9.Dockerfile create mode 100644 cpython-unix/build.debian9.Dockerfile create mode 100644 cpython-unix/gcc.debian9.Dockerfile create mode 100644 cpython-unix/xcb.debian9.Dockerfile diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c0c21bd0..0039922b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,7 +2,7 @@ name: linux on: push: - branches: [main] + branches: [ main ] pull_request: concurrency: @@ -56,15 +56,28 @@ jobs: fail-fast: false matrix: image: - - build - - build.cross - - build.cross-riscv64 - - gcc - - xcb - - xcb.cross - - xcb.cross-riscv64 - name: ${{ matrix.image }} - runs-on: depot-ubuntu-22.04 + - name: build + platform: linux64 + - name: build.cross + platform: linux64 + - name: build.cross-riscv64 + platform: linux64 + - name: build.debian9 + platform: linux_aarch64 + - name: gcc + platform: linux64 + - name: gcc.debian9 + platform: linux_aarch64 + - name: xcb + platform: linux64 + - name: xcb.cross + platform: linux64 + - name: xcb.cross-riscv64 + platform: linux64 + - name: xcb.debian9 + platform: linux_aarch64 + name: ${{ matrix.image.name }} + runs-on: ${{ matrix.image.platform == 'linux_aarch64' && 'depot-ubuntu-22.04-arm' || 'depot-ubuntu-22.04' }} permissions: packages: write steps: @@ -98,30 +111,30 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: build/${{ matrix.image }}.Dockerfile + file: build/${{ matrix.image.name }}.Dockerfile labels: org.opencontainers.image.source=https://github.com/${{ env.REPO_NAME }} # Cache from/to the current branch of the current repo as the primary cache key. # Cache from the default branch of the current repo so branches can have cache hits. # Cache from the default branch of the canonical repo so forks can have cache hits. # Ignore errors on cache writes so CI of forks works without a valid GHCR config. cache-from: | - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }} - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-main - type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.image }}-main + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image.name }}-${{ matrix.image.platform }}-${{ env.GIT_REF_NAME }} + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image.name }}-${{ matrix.image.platform }}-main + type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.image.name }}-${{ matrix.image.platform }}-main cache-to: | - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }},ignore-error=true + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image.name }}-${{ matrix.image.platform }}-${{ env.GIT_REF_NAME }},ignore-error=true outputs: | - type=docker,dest=build/image-${{ matrix.image }}.tar + type=docker,dest=build/image-${{ matrix.image.name }}.${{ matrix.image.platform }}.tar - name: Compress Image run: | - echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image }} + echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image.name }}.${{ matrix.image.platform }} zstd -v -T0 -6 --rm build/image-*.tar - name: Upload Docker Image uses: actions/upload-artifact@v4 with: - name: image-${{ matrix.image }} + name: image-${{ matrix.image.name }}-${{ matrix.image.platform }} path: build/image-* generate-matrix: diff --git a/ci-runners.yaml b/ci-runners.yaml index aeb0e5ca..e5598fb1 100644 --- a/ci-runners.yaml +++ b/ci-runners.yaml @@ -5,11 +5,10 @@ depot-ubuntu-22.04: platform: linux free: false -# TODO: Enable this runner to perform native builds for aarch64 -# depot-ubuntu-22.04-arm: -# arch: aarch64 -# platform: linux -# free: false +depot-ubuntu-22.04-arm: + arch: aarch64 + platform: linux + free: false depot-macos-latest: arch: x86_64 diff --git a/ci-targets.yaml b/ci-targets.yaml index 9a3dac93..f4f12921 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -13,12 +13,10 @@ linux: - "3.14" build_options: - debug - - noopt - - lto + - pgo+lto build_options_conditional: - options: - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto + - freethreaded+pgo+lto minimum-python-version: "3.13" \ No newline at end of file diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 196b6bbf..e0a24bdd 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -66,19 +66,19 @@ TOOLCHAIN_DEPENDS := \ PYTHON_DEP_DEPENDS := \ $(OUTDIR)/targets/$(TARGET_TRIPLE) \ - $(if $(PYBUILD_NO_DOCKER),,$(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).tar) \ + $(if $(PYBUILD_NO_DOCKER),,$(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).$(HOST_PLATFORM).tar) \ $(TOOLCHAIN_DEPENDS) \ $(NULL) default: $(OUTDIR)/cpython-$(CPYTHON_$(PYTHON_MAJOR_VERSION)_VERSION)-$(PACKAGE_SUFFIX).tar ifndef PYBUILD_NO_DOCKER -$(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile +$(OUTDIR)/image-%.$(HOST_PLATFORM).tar: $(OUTDIR)/%.Dockerfile $(RUN_BUILD) --toolchain image-$* endif -$(OUTDIR)/binutils-$(BINUTILS_VERSION)-$(HOST_PLATFORM).tar: $(OUTDIR)/image-gcc.tar $(HERE)/build-binutils.sh - $(RUN_BUILD) --toolchain binutils +$(OUTDIR)/binutils-$(BINUTILS_VERSION)-$(HOST_PLATFORM).tar: $(HERE)/build-binutils.sh + $(RUN_BUILD) --toolchain --docker-image $(DOCKER_IMAGE_GCC) binutils $(OUTDIR)/$(CLANG_FILENAME): $(RUN_BUILD) --toolchain clang --target-triple $(TARGET_TRIPLE) @@ -123,7 +123,7 @@ $(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_D $(OUTDIR)/libffi-$(LIBFFI_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libffi.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) libffi -$(OUTDIR)/libpthread-stubs-$(LIBPTHREAD_STUBS_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libpthread-stubs.sh $(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).tar +$(OUTDIR)/libpthread-stubs-$(LIBPTHREAD_STUBS_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libpthread-stubs.sh $(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).$(HOST_PLATFORM).tar $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) libpthread-stubs LIBX11_DEPENDS = \ @@ -150,7 +150,7 @@ $(OUTDIR)/libXau-$(LIBXAU_VERSION)-$(PACKAGE_SUFFIX).tar: $(LIBXAU_DEPENDS) LIBXCB_DEPENDS = \ $(PYTHON_DEP_DEPENDS) \ $(HERE)/build-libxcb.sh \ - $(OUTDIR)/image-$(DOCKER_IMAGE_XCB).tar \ + $(OUTDIR)/image-$(DOCKER_IMAGE_XCB).$(HOST_PLATFORM).tar \ $(OUTDIR)/xcb-proto-$(XCB_PROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/libXau-$(LIBXAU_VERSION)-$(PACKAGE_SUFFIX).tar \ $(OUTDIR)/xorgproto-$(XORGPROTO_VERSION)-$(PACKAGE_SUFFIX).tar \ diff --git a/cpython-unix/base.debian9.Dockerfile b/cpython-unix/base.debian9.Dockerfile new file mode 100644 index 00000000..1b47b35a --- /dev/null +++ b/cpython-unix/base.debian9.Dockerfile @@ -0,0 +1,38 @@ +# Debian Stretch. +FROM debian@sha256:c5c5200ff1e9c73ffbf188b4a67eb1c91531b644856b4aefe86a58d2f0cb05be +MAINTAINER Gregory Szorc + +RUN groupadd -g 1000 build && \ + useradd -u 1000 -g 1000 -d /build -s /bin/bash -m build && \ + mkdir /tools && \ + chown -R build:build /build /tools + +ENV HOME=/build \ + SHELL=/bin/bash \ + USER=build \ + LOGNAME=build \ + HOSTNAME=builder \ + DEBIAN_FRONTEND=noninteractive + +CMD ["/bin/bash", "--login"] +WORKDIR '/build' + +RUN for s in debian_stretch debian_stretch-updates debian-security_stretch/updates; do \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/20230423T032736Z/ ${s#*_} main"; \ + done > /etc/apt/sources.list && \ + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + ) > /etc/apt/apt.conf.d/99cpython-portable + +# apt iterates all available file descriptors up to rlim_max and calls +# fcntl(fd, F_SETFD, FD_CLOEXEC). This can result in millions of system calls +# (we've seen 1B in the wild) and cause operations to take seconds to minutes. +# Setting a fd limit mitigates. +# +# Attempts at enforcing the limit globally via /etc/security/limits.conf and +# /root/.bashrc were not successful. Possibly because container image builds +# don't perform a login or use a shell the way we expect. +RUN ulimit -n 10000 && apt-get update diff --git a/cpython-unix/build-binutils.sh b/cpython-unix/build-binutils.sh index b287e2e1..a45adee0 100755 --- a/cpython-unix/build-binutils.sh +++ b/cpython-unix/build-binutils.sh @@ -22,9 +22,15 @@ if [ -x "${SCCACHE}" ]; then export STAGE_CC_WRAPPER="${SCCACHE}" fi +if [ "$(uname -m)" = "x86_64" ]; then + triple="x86_64-unknown-linux-gnu" +else + triple="aarch64-unknown-linux-gnu" +fi + # gprofng requires a bison newer than what we have. So just disable it. ../binutils-${BINUTILS_VERSION}/configure \ - --build=x86_64-unknown-linux-gnu \ + --build=${triple} \ --prefix=/tools/host \ --enable-plugins \ --enable-gprofng=no \ diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 6bf91b49..70926002 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -29,8 +29,13 @@ def main(): if sys.platform == "linux": - host_platform = "linux64" - default_target_triple = "x86_64-unknown-linux-gnu" + machine = platform.machine() + if machine == "aarch64": + host_platform = "linux_aarch64" + default_target_triple = "aarch64-unknown-linux-gnu" + else: + host_platform = "linux64" + default_target_triple = "x86_64-unknown-linux-gnu" elif sys.platform == "darwin": host_platform = "macos" machine = platform.machine() @@ -109,10 +114,12 @@ def main(): "toolchain-image-build", "toolchain-image-build.cross", "toolchain-image-build.cross-riscv64", + "toolchain-image-build.debian9", "toolchain-image-gcc", "toolchain-image-xcb", "toolchain-image-xcb.cross", "toolchain-image-xcb.cross-riscv64", + "toolchain-image-xcb.debian9", }, default="default", help="The make target to evaluate", diff --git a/cpython-unix/build.debian9.Dockerfile b/cpython-unix/build.debian9.Dockerfile new file mode 100644 index 00000000..1f7c91b6 --- /dev/null +++ b/cpython-unix/build.debian9.Dockerfile @@ -0,0 +1,16 @@ +{% include 'base.debian9.Dockerfile' %} + +RUN ulimit -n 10000 && apt-get install \ + bzip2 \ + file \ + libc6-dev \ + libffi-dev \ + make \ + patch \ + perl \ + pkg-config \ + tar \ + xz-utils \ + unzip \ + zip \ + zlib1g-dev diff --git a/cpython-unix/build.py b/cpython-unix/build.py index f058e93a..bae4ba22 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -117,13 +117,17 @@ def add_target_env(env, build_platform, target_triple, build_env): extra_host_ldflags = [] if build_platform == "linux64": - env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + machine = platform.machine() - env["TARGET_TRIPLE"] = ( - target_triple.replace("x86_64_v2-", "x86_64-") - .replace("x86_64_v3-", "x86_64-") - .replace("x86_64_v4-", "x86_64-") - ) + if machine == "aarch64": + env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" + else: + env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + env["TARGET_TRIPLE"] = ( + target_triple.replace("x86_64_v2-", "x86_64-") + .replace("x86_64_v3-", "x86_64-") + .replace("x86_64_v4-", "x86_64-") + ) # This will make x86_64_v2, etc count as cross-compiling. This is # semantically correct, since the current machine may not support @@ -133,6 +137,10 @@ def add_target_env(env, build_platform, target_triple, build_env): ): env["CROSS_COMPILING"] = "1" + elif build_platform == "linux_aarch64": + env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" + env["TARGET_TRIPLE"] = "aarch64-unknown-linux-gnu" + if build_platform == "macos": machine = platform.machine() @@ -516,10 +524,12 @@ def python_build_info( binary_suffix = "" - if platform == "linux64": + if platform in ("linux64", "linux_aarch64"): + arch = "aarch64" if platform == "linux_aarch64" else "x86_64" + bi["core"]["static_lib"] = ( - "install/lib/python{version}/config-{version}{binary_suffix}-x86_64-linux-gnu/libpython{version}{binary_suffix}.a".format( - version=version, binary_suffix=binary_suffix + "install/lib/python{version}/config-{version}{binary_suffix}-{arch}-linux-gnu/libpython{version}{binary_suffix}.a".format( + version=version, binary_suffix=binary_suffix, arch=arch, ) ) @@ -599,7 +609,7 @@ def python_build_info( if lib.startswith("-l"): lib = lib[2:] - if platform == "linux64" and lib not in linux_allowed_system_libraries: + if platform in ("linux64", "linux_aarch64") and lib not in linux_allowed_system_libraries: raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib)) elif platform == "macos" and lib not in MACOS_ALLOW_SYSTEM_LIBRARIES: raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib)) @@ -867,7 +877,7 @@ def build_cpython( extension_module_loading = ["builtin"] crt_features = [] - if host_platform == "linux64": + if host_platform in ("linux64", "linux_aarch64"): if "static" in parsed_build_options: crt_features.append("static") else: @@ -1089,10 +1099,10 @@ def main(): with image_path.open("rb") as fh: image_data = fh.read() - build_docker_image(client, image_data, BUILD, image_name) + build_docker_image(client, image_data, BUILD, image_name, host_platform) elif action == "binutils": - build_binutils(client, get_image(client, ROOT, BUILD, "gcc"), host_platform) + build_binutils(client, get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform) elif action == "clang": materialize_clang(host_platform, target_triple) @@ -1100,7 +1110,7 @@ def main(): elif action == "musl": build_musl( client, - get_image(client, ROOT, BUILD, "gcc"), + get_image(client, ROOT, BUILD, "gcc", host_platform), host_platform, target_triple, build_options, @@ -1110,7 +1120,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1124,7 +1134,7 @@ def main(): build_libedit( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform=host_platform, target_triple=target_triple, build_options=build_options, @@ -1158,7 +1168,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1171,7 +1181,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1191,7 +1201,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1204,7 +1214,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1216,7 +1226,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1229,7 +1239,7 @@ def main(): build_tix( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform=host_platform, target_triple=target_triple, build_options=build_options, @@ -1250,7 +1260,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1262,7 +1272,7 @@ def main(): elif action.startswith("cpython-") and action.endswith("-host"): build_cpython_host( client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action[:-5], host_platform=host_platform, target_triple=target_triple, @@ -1281,7 +1291,7 @@ def main(): build_cpython( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform=host_platform, target_triple=target_triple, build_options=build_options, diff --git a/cpython-unix/gcc.debian9.Dockerfile b/cpython-unix/gcc.debian9.Dockerfile new file mode 100644 index 00000000..92d764e8 --- /dev/null +++ b/cpython-unix/gcc.debian9.Dockerfile @@ -0,0 +1,14 @@ +{% include 'base.debian9.Dockerfile' %} +RUN ulimit -n 10000 && apt-get install \ + autoconf \ + automake \ + bison \ + build-essential \ + gawk \ + gcc \ + libtool \ + make \ + tar \ + texinfo \ + xz-utils \ + unzip diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 1b179768..8ba65c86 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -153,7 +153,7 @@ aarch64-apple-ios: aarch64-unknown-linux-gnu: host_platforms: - - linux64 + - linux_aarch64 pythons_supported: - '3.9' - '3.10' @@ -161,11 +161,16 @@ aarch64-unknown-linux-gnu: - '3.12' - '3.13' - '3.14' - docker_image_suffix: .cross - host_cc: /usr/bin/x86_64-linux-gnu-gcc - host_cxx: /usr/bin/x86_64-linux-gnu-g++ - target_cc: /usr/bin/aarch64-linux-gnu-gcc - target_cxx: /usr/bin/aarch64-linux-gnu-g++ + docker_image_suffix: .debian9 + needs_toolchain: true + host_cc: clang + host_cxx: clang++ + target_cc: clang + target_cxx: clang++ + target_cflags: + - '-fvisibility=hidden' + # Needed to prevent BOLT from crashing. + - '-fdebug-default-version=4' needs: - autoconf - bdb @@ -191,6 +196,10 @@ aarch64-unknown-linux-gnu: - xz - zlib openssl_target: linux-aarch64 + # Blocked on: + # BOLT-ERROR: Cannot relax adr in non-simple function + # trampoline_code_table/1. Use --strict option to override + # bolt_capable: true arm64-apple-tvos: host_platforms: diff --git a/cpython-unix/xcb.debian9.Dockerfile b/cpython-unix/xcb.debian9.Dockerfile new file mode 100644 index 00000000..8ade8a12 --- /dev/null +++ b/cpython-unix/xcb.debian9.Dockerfile @@ -0,0 +1,3 @@ +{% include 'build.debian9.Dockerfile' %} +RUN ulimit -n 10000 && apt-get install \ + python diff --git a/docs/running.rst b/docs/running.rst index d2d2dc9a..c2393de0 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -143,7 +143,7 @@ Common configurations include: Profile guided optimization. Starting with CPython 3.12, BOLT is also applied alongside traditional - PGO on platforms supporting BOLT. (Currently just Linux x86-64.) + PGO on platforms supporting BOLT. (Currently just Linux aarch64 and x86-64.) ``lto`` Link-time optimization. diff --git a/pythonbuild/docker.py b/pythonbuild/docker.py index 4269b2bd..af7ff5b1 100644 --- a/pythonbuild/docker.py +++ b/pythonbuild/docker.py @@ -29,8 +29,8 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path): write_if_different(dest_dir / f, data.encode("utf-8")) -def build_docker_image(client, image_data: bytes, image_dir: pathlib.Path, name): - image_path = image_dir / ("image-%s" % name) +def build_docker_image(client, image_data: bytes, image_dir: pathlib.Path, name, host_platform): + image_path = image_dir / f"image-{name}.{host_platform}" return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path) @@ -66,11 +66,12 @@ def ensure_docker_image(client, fh, image_path=None): return image -def get_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name): +def get_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name, host_platform): if client is None: return None - image_path = image_dir / ("image-%s" % name) + image_name = f"image-{name}.{host_platform}" + image_path = image_dir / image_name tar_path = image_path.with_suffix(".tar") with image_path.open("r") as fh: @@ -88,7 +89,7 @@ def get_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name): return image_id else: - return build_docker_image(client, str(source_dir).encode(), image_dir, name) + return build_docker_image(client, str(source_dir).encode(), image_dir, name, host_platform) def copy_file_to_container(path, container, container_path, archive_path=None): diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index b777f748..446ebbeb 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -174,6 +174,13 @@ "version": "14.0.3+20220508", }, # Remember to update LLVM_URL in src/release.rs whenever upgrading. + "llvm-20-aarch64-linux": { + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-aarch64-unknown-linux-gnu.tar.zst", + "size": 255946687, + "sha256": "e70753f294b8f83fffbaf07af36857c27ceaef0291cb10f724ada6af11b0a5bc", + "version": "20.1.4+20250511", + }, + # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-20-x86_64-linux": { "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-x86_64-unknown-linux-gnu.tar.zst", "size": 299883811, diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index 14d9a474..7d071e66 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -49,6 +49,8 @@ def supported_targets(yaml_path: pathlib.Path): for host_platform in settings["host_platforms"]: if sys.platform == "linux" and host_platform == "linux64": targets.add(target) + elif sys.platform == "linux" and host_platform == "linux_aarch64": + targets.add(target) elif sys.platform == "darwin" and host_platform == "macos": targets.add(target) @@ -160,6 +162,7 @@ def write_triples_makefiles( image_suffix = settings.get("docker_image_suffix", "") lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_GCC := gcc%s\n" % image_suffix) lines.append("DOCKER_IMAGE_XCB := xcb%s\n" % image_suffix) entry = clang_toolchain(host_platform, triple) @@ -436,6 +439,8 @@ def clang_toolchain(host_platform: str, target_triple: str) -> str: return "llvm-14-x86_64-linux" else: return "llvm-20-x86_64-linux" + elif host_platform == "linux_aarch64": + return "llvm-20-aarch64-linux" elif host_platform == "macos": if platform.mac_ver()[2] == "arm64": return "llvm-aarch64-macos"