From 695c72cb213de08c8380143328769b67ac10c7b1 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 19:13:29 -0700 Subject: [PATCH 1/4] xtrans 1.5.0 -> 1.6.0 This seems to "just work." Let's stay modern. --- pythonbuild/downloads.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 3e7adc82..0367faa8 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -392,10 +392,10 @@ "version": "2019.1", }, "xtrans": { - "url": "https://www.x.org/archive/individual/lib/xtrans-1.5.0.tar.gz", - "size": 230197, - "sha256": "a806f8a92f879dcd0146f3f1153fdffe845f2fc0df9b1a26c19312b7b0a29c86", - "version": "1.5.0", + "url": "https://www.x.org/archive/individual/lib/xtrans-1.6.0.tar.gz", + "size": 239113, + "sha256": "936b74c60b19c317c3f3cb1b114575032528dbdaf428740483200ea874c2ca0a", + "version": "1.6.0", }, # IMPORTANT: xz 5.6 has a backdoor. Be extremely cautious before taking any xz # upgrade since it isn't clear which versions are safe. From b40be4631bc7bb246cd6723613c6d5cf3e0910ce Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 19:14:46 -0700 Subject: [PATCH 2/4] x11-util-macros 1.20.1 -> 1.20.2 --- pythonbuild/downloads.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 0367faa8..3d888e88 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -372,10 +372,10 @@ "license_file": "LICENSE.libuuid.txt", }, "x11-util-macros": { - "url": "https://www.x.org/archive/individual/util/util-macros-1.20.1.tar.gz", - "size": 105481, - "sha256": "b373f72887b1394ce2193180a60cb0d1fb8b17bc96ddd770cfd7a808cb489a15", - "version": "1.20.1", + "url": "https://www.x.org/archive/individual/util/util-macros-1.20.2.tar.gz", + "size": 105410, + "sha256": "f642f8964d81acdf06653fdf9dbc210c43ce4bd308bd644a8d573148d0ced76b", + "version": "1.20.2", }, "xcb-proto": { "url": "https://xcb.freedesktop.org/dist/xcb-proto-1.17.0.tar.xz", From 0211a8c21c97901dfe763a93a35c81eaa47ef1a2 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 16 May 2025 19:18:59 -0700 Subject: [PATCH 3/4] xorgproto 2019.1 -> 2024.1 I think deletion of the legacy proto packages a few commits ago fixed up the build failures the deleted comment alluded to. --- pythonbuild/downloads.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index 3d888e88..c1cae140 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -383,13 +383,11 @@ "sha256": "2c1bacd2110f4799f74de6ebb714b94cf6f80fb112316b1219480fd22562148c", "version": "1.17.0", }, - # Newer versions from at least 2023 have build failures for reasons we haven't - # fully investigated. "xorgproto": { - "url": "https://www.x.org/archive/individual/proto/xorgproto-2019.1.tar.gz", - "size": 1119813, - "sha256": "38ad1d8316515785d53c5162b4b7022918e03c11d72a5bd9df0a176607f42bca", - "version": "2019.1", + "url": "https://www.x.org/archive/individual/proto/xorgproto-2024.1.tar.gz", + "size": 1115486, + "sha256": "4f6b9b4faf91e5df8265b71843a91fc73dc895be6210c84117a996545df296ce", + "version": "2024.1", }, "xtrans": { "url": "https://www.x.org/archive/individual/lib/xtrans-1.6.0.tar.gz", From 285e3072d9b5c2ef6e3c8467a56712e92f13197f Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sat, 28 Jun 2025 14:04:53 -0500 Subject: [PATCH 4/4] Enable native builds on Linux aarch64 I just published an LLVM 20 toolchain for aarch64. The toolchain has support for PGO and BOLT. This commit switches the Linux aarch64 builds to be performed natively on aarch64 machines. PGO and BOLT are enabled on the builds, hopefully making them a bit faster. --- .github/workflows/linux.yml | 18 +++---- ci-matrix.py | 2 + ci-runners.yaml | 9 ++-- ci-targets.yaml | 6 +-- cpython-unix/Makefile | 12 ++--- cpython-unix/base.debian9.Dockerfile | 38 ++++++++++++++ cpython-unix/build-binutils.sh | 8 ++- cpython-unix/build-main.py | 1 + cpython-unix/build.debian9.Dockerfile | 16 ++++++ cpython-unix/build.py | 74 +++++++++++++++++---------- cpython-unix/gcc.debian9.Dockerfile | 14 +++++ cpython-unix/targets.yml | 21 ++++++-- pythonbuild/docker.py | 17 ++++-- pythonbuild/downloads.py | 7 +++ pythonbuild/utils.py | 14 +++++ 15 files changed, 195 insertions(+), 62 deletions(-) create mode 100644 cpython-unix/base.debian9.Dockerfile create mode 100644 cpython-unix/build.debian9.Dockerfile create mode 100644 cpython-unix/gcc.debian9.Dockerfile diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 8ae87e76..d4602fbe 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -2,7 +2,7 @@ name: linux on: push: - branches: [main] + branches: [ main ] pull_request: concurrency: @@ -58,7 +58,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs.docker-build-matrix) }} - name: image / ${{ matrix.name }} + name: image / ${{ matrix.arch }} / ${{ matrix.name }} runs-on: ${{ matrix.runner }} permissions: packages: write @@ -100,23 +100,23 @@ jobs: # Cache from the default branch of the canonical repo so forks can have cache hits. # Ignore errors on cache writes so CI of forks works without a valid GHCR config. cache-from: | - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-${{ env.GIT_REF_NAME }} - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-main - type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.name }}-main + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-linux_${{ matrix.arch }}-${{ env.GIT_REF_NAME }} + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-linux_${{ matrix.arch }}-main + type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.name }}-linux_${{ matrix.arch }}-main cache-to: | - type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-${{ env.GIT_REF_NAME }},ignore-error=true + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-linux_${{ matrix.arch }}-${{ env.GIT_REF_NAME }},ignore-error=true outputs: | - type=docker,dest=build/image-${{ matrix.name }}.tar + type=docker,dest=build/image-${{ matrix.name }}.linux_${{ matrix.arch }}.tar - name: Compress Image run: | - echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.name }} + echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.name }}.linux_${{ matrix.arch }} zstd -v -T0 -6 --rm build/image-*.tar - name: Upload Docker Image uses: actions/upload-artifact@v4 with: - name: image-${{ matrix.name }} + name: image-${{ matrix.name }}-linux_${{ matrix.arch }} path: build/image-* generate-matrix: diff --git a/ci-matrix.py b/ci-matrix.py index cfccdd5e..91cfa953 100644 --- a/ci-matrix.py +++ b/ci-matrix.py @@ -25,7 +25,9 @@ {"name": "build", "arch": "x86_64"}, {"name": "build.cross", "arch": "x86_64"}, {"name": "build.cross-riscv64", "arch": "x86_64"}, + {"name": "build.debian9", "arch": "aarch64"}, {"name": "gcc", "arch": "x86_64"}, + {"name": "gcc.debian9", "arch": "aarch64"}, ] diff --git a/ci-runners.yaml b/ci-runners.yaml index 43ff9414..dae6aabc 100644 --- a/ci-runners.yaml +++ b/ci-runners.yaml @@ -5,11 +5,10 @@ depot-ubuntu-22.04: platform: linux free: false -# TODO: Enable this runner to perform native builds for aarch64 -# depot-ubuntu-22.04-arm: -# arch: aarch64 -# platform: linux -# free: false +depot-ubuntu-22.04-arm: + arch: aarch64 + platform: linux + free: false depot-macos-latest: arch: x86_64 diff --git a/ci-targets.yaml b/ci-targets.yaml index c7b4bd12..c59d5b7f 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -50,13 +50,11 @@ linux: - "3.14" build_options: - debug - - noopt - - lto + - pgo+lto build_options_conditional: - options: - freethreaded+debug - - freethreaded+noopt - - freethreaded+lto + - freethreaded+pgo+lto minimum-python-version: "3.13" armv7-unknown-linux-gnueabi: diff --git a/cpython-unix/Makefile b/cpython-unix/Makefile index 4cacaff3..6e4b0070 100644 --- a/cpython-unix/Makefile +++ b/cpython-unix/Makefile @@ -66,7 +66,7 @@ TOOLCHAIN_DEPENDS := \ PYTHON_DEP_DEPENDS := \ $(OUTDIR)/targets/$(TARGET_TRIPLE) \ - $(if $(PYBUILD_NO_DOCKER),,$(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).tar) \ + $(if $(PYBUILD_NO_DOCKER),,$(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).$(HOST_PLATFORM).tar) \ $(TOOLCHAIN_DEPENDS) \ $(NULL) @@ -75,18 +75,18 @@ HOST_PYTHON_DEPENDS := $(OUTDIR)/cpython-$(PYTHON_MAJOR_VERSION)-$(CPYTHON_$(PYT default: $(OUTDIR)/cpython-$(CPYTHON_$(PYTHON_MAJOR_VERSION)_VERSION)-$(PACKAGE_SUFFIX).tar ifndef PYBUILD_NO_DOCKER -$(OUTDIR)/image-%.tar: $(OUTDIR)/%.Dockerfile +$(OUTDIR)/image-%.$(HOST_PLATFORM).tar: $(OUTDIR)/%.Dockerfile $(RUN_BUILD) --toolchain image-$* endif -$(OUTDIR)/binutils-$(BINUTILS_VERSION)-$(HOST_PLATFORM).tar: $(OUTDIR)/image-gcc.tar $(HERE)/build-binutils.sh - $(RUN_BUILD) --toolchain binutils +$(OUTDIR)/binutils-$(BINUTILS_VERSION)-$(HOST_PLATFORM).tar: $(HERE)/build-binutils.sh + $(RUN_BUILD) --toolchain --docker-image $(DOCKER_IMAGE_GCC) binutils $(OUTDIR)/$(CLANG_FILENAME): $(RUN_BUILD) --toolchain clang --target-triple $(TARGET_TRIPLE) $(OUTDIR)/musl-$(MUSL_VERSION)-$(HOST_PLATFORM).tar: $(BASE_TOOLCHAIN_DEPENDS) $(HERE)/build-musl.sh - $(RUN_BUILD) --toolchain musl + $(RUN_BUILD) --toolchain musl --docker-image $(DOCKER_IMAGE_GCC) ifeq ($(HOST_PLATFORM),linux_x86_64) TOOLCHAIN_TARGET := $(OUTDIR)/musl-$(MUSL_VERSION)-$(HOST_PLATFORM).tar @@ -125,7 +125,7 @@ $(OUTDIR)/libffi-3.3-$(LIBFFI_3.3_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_D $(OUTDIR)/libffi-$(LIBFFI_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libffi.sh $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) libffi -$(OUTDIR)/libpthread-stubs-$(LIBPTHREAD_STUBS_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libpthread-stubs.sh $(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).tar +$(OUTDIR)/libpthread-stubs-$(LIBPTHREAD_STUBS_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-libpthread-stubs.sh $(OUTDIR)/image-$(DOCKER_IMAGE_BUILD).$(HOST_PLATFORM).tar $(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) libpthread-stubs LIBX11_DEPENDS = \ diff --git a/cpython-unix/base.debian9.Dockerfile b/cpython-unix/base.debian9.Dockerfile new file mode 100644 index 00000000..1b47b35a --- /dev/null +++ b/cpython-unix/base.debian9.Dockerfile @@ -0,0 +1,38 @@ +# Debian Stretch. +FROM debian@sha256:c5c5200ff1e9c73ffbf188b4a67eb1c91531b644856b4aefe86a58d2f0cb05be +MAINTAINER Gregory Szorc + +RUN groupadd -g 1000 build && \ + useradd -u 1000 -g 1000 -d /build -s /bin/bash -m build && \ + mkdir /tools && \ + chown -R build:build /build /tools + +ENV HOME=/build \ + SHELL=/bin/bash \ + USER=build \ + LOGNAME=build \ + HOSTNAME=builder \ + DEBIAN_FRONTEND=noninteractive + +CMD ["/bin/bash", "--login"] +WORKDIR '/build' + +RUN for s in debian_stretch debian_stretch-updates debian-security_stretch/updates; do \ + echo "deb http://snapshot.debian.org/archive/${s%_*}/20230423T032736Z/ ${s#*_} main"; \ + done > /etc/apt/sources.list && \ + ( echo 'quiet "true";'; \ + echo 'APT::Get::Assume-Yes "true";'; \ + echo 'APT::Install-Recommends "false";'; \ + echo 'Acquire::Check-Valid-Until "false";'; \ + echo 'Acquire::Retries "5";'; \ + ) > /etc/apt/apt.conf.d/99cpython-portable + +# apt iterates all available file descriptors up to rlim_max and calls +# fcntl(fd, F_SETFD, FD_CLOEXEC). This can result in millions of system calls +# (we've seen 1B in the wild) and cause operations to take seconds to minutes. +# Setting a fd limit mitigates. +# +# Attempts at enforcing the limit globally via /etc/security/limits.conf and +# /root/.bashrc were not successful. Possibly because container image builds +# don't perform a login or use a shell the way we expect. +RUN ulimit -n 10000 && apt-get update diff --git a/cpython-unix/build-binutils.sh b/cpython-unix/build-binutils.sh index c52e8238..2bb89364 100755 --- a/cpython-unix/build-binutils.sh +++ b/cpython-unix/build-binutils.sh @@ -11,9 +11,15 @@ tar -xf binutils-${BINUTILS_VERSION}.tar.xz mkdir binutils-objdir pushd binutils-objdir +if [ "$(uname -m)" = "x86_64" ]; then + triple="x86_64-unknown-linux-gnu" +else + triple="aarch64-unknown-linux-gnu" +fi + # gprofng requires a bison newer than what we have. So just disable it. ../binutils-${BINUTILS_VERSION}/configure \ - --build=x86_64-unknown-linux-gnu \ + --build=${triple} \ --prefix=/tools/host \ --enable-plugins \ --enable-gprofng=no \ diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index a353daf2..7a7d08b1 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -95,6 +95,7 @@ def main(): "toolchain-image-build", "toolchain-image-build.cross", "toolchain-image-build.cross-riscv64", + "toolchain-image-build.debian9", "toolchain-image-gcc", "toolchain-image-xcb", "toolchain-image-xcb.cross", diff --git a/cpython-unix/build.debian9.Dockerfile b/cpython-unix/build.debian9.Dockerfile new file mode 100644 index 00000000..1f7c91b6 --- /dev/null +++ b/cpython-unix/build.debian9.Dockerfile @@ -0,0 +1,16 @@ +{% include 'base.debian9.Dockerfile' %} + +RUN ulimit -n 10000 && apt-get install \ + bzip2 \ + file \ + libc6-dev \ + libffi-dev \ + make \ + patch \ + perl \ + pkg-config \ + tar \ + xz-utils \ + unzip \ + zip \ + zlib1g-dev diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 6f98e990..01d39b2a 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -88,14 +88,21 @@ def add_target_env(env, build_platform, target_triple, build_env): extra_host_cflags = [] extra_host_ldflags = [] - if build_platform == "linux_x86_64": - env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + if build_platform.startswith("linux_"): + machine = platform.machine() - env["TARGET_TRIPLE"] = ( - target_triple.replace("x86_64_v2-", "x86_64-") - .replace("x86_64_v3-", "x86_64-") - .replace("x86_64_v4-", "x86_64-") - ) + if machine == "aarch64": + env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" + env["TARGET_TRIPLE"] = target_triple + elif machine == "x86_64": + env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + env["TARGET_TRIPLE"] = ( + target_triple.replace("x86_64_v2-", "x86_64-") + .replace("x86_64_v3-", "x86_64-") + .replace("x86_64_v4-", "x86_64-") + ) + else: + raise Exception("unhandled Linux machine value: %s" % machine) # This will make x86_64_v2, etc count as cross-compiling. This is # semantically correct, since the current machine may not support @@ -105,7 +112,7 @@ def add_target_env(env, build_platform, target_triple, build_env): ): env["CROSS_COMPILING"] = "1" - if build_platform.startswith("macos_"): + elif build_platform.startswith("macos_"): machine = platform.machine() if machine == "arm64": @@ -188,6 +195,8 @@ def add_target_env(env, build_platform, target_triple, build_env): extra_host_cflags.extend(["-isysroot", host_sdk_path]) extra_host_ldflags.extend(["-isysroot", host_sdk_path]) + else: + raise Exception("unhandled build platform: %s" % build_platform) env["EXTRA_HOST_CFLAGS"] = " ".join(extra_host_cflags) env["EXTRA_HOST_LDFLAGS"] = " ".join(extra_host_ldflags) @@ -496,10 +505,14 @@ def python_build_info( binary_suffix = "" - if platform == "linux_x86_64": + if platform in ("linux_x86_64", "linux_aarch64"): + arch = platform.removeprefix("linux_") + bi["core"]["static_lib"] = ( - "install/lib/python{version}/config-{version}{binary_suffix}-x86_64-linux-gnu/libpython{version}{binary_suffix}.a".format( - version=version, binary_suffix=binary_suffix + "install/lib/python{version}/config-{version}{binary_suffix}-{arch}-linux-gnu/libpython{version}{binary_suffix}.a".format( + version=version, + binary_suffix=binary_suffix, + arch=arch, ) ) @@ -579,7 +592,10 @@ def python_build_info( if lib.startswith("-l"): lib = lib[2:] - if platform == "linux_x86_64" and lib not in linux_allowed_system_libraries: + if ( + platform in ("linux_x86_64", "linux_aarch64") + and lib not in linux_allowed_system_libraries + ): raise Exception("unexpected library in LIBS (%s): %s" % (libs, lib)) elif ( platform.startswith("macos_") @@ -851,7 +867,7 @@ def build_cpython( extension_module_loading = ["builtin"] crt_features = [] - if host_platform == "linux_x86_64": + if host_platform in ("linux_x86_64", "linux_aarch64"): if "static" in parsed_build_options: crt_features.append("static") else: @@ -1080,10 +1096,14 @@ def main(): with image_path.open("rb") as fh: image_data = fh.read() - build_docker_image(client, image_data, BUILD, image_name) + build_docker_image(client, image_data, BUILD, image_name, host_platform) elif action == "binutils": - build_binutils(client, get_image(client, ROOT, BUILD, "gcc"), host_platform) + build_binutils( + client, + get_image(client, ROOT, BUILD, docker_image, host_platform), + host_platform, + ) elif action == "clang": materialize_clang(host_platform, target_triple) @@ -1091,7 +1111,7 @@ def main(): elif action == "musl": build_musl( client, - get_image(client, ROOT, BUILD, "gcc"), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform, target_triple, build_options, @@ -1101,7 +1121,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1115,7 +1135,7 @@ def main(): build_libedit( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform=host_platform, target_triple=target_triple, build_options=build_options, @@ -1149,7 +1169,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1162,7 +1182,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1182,7 +1202,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1195,7 +1215,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1208,7 +1228,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1222,7 +1242,7 @@ def main(): build_tix( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform=host_platform, target_triple=target_triple, build_options=build_options, @@ -1243,7 +1263,7 @@ def main(): simple_build( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action, host_platform=host_platform, target_triple=target_triple, @@ -1256,7 +1276,7 @@ def main(): elif action.startswith("cpython-") and action.endswith("-host"): build_cpython_host( client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), action[:-5], host_platform=host_platform, target_triple=target_triple, @@ -1275,7 +1295,7 @@ def main(): build_cpython( settings, client, - get_image(client, ROOT, BUILD, docker_image), + get_image(client, ROOT, BUILD, docker_image, host_platform), host_platform=host_platform, target_triple=target_triple, build_options=build_options, diff --git a/cpython-unix/gcc.debian9.Dockerfile b/cpython-unix/gcc.debian9.Dockerfile new file mode 100644 index 00000000..92d764e8 --- /dev/null +++ b/cpython-unix/gcc.debian9.Dockerfile @@ -0,0 +1,14 @@ +{% include 'base.debian9.Dockerfile' %} +RUN ulimit -n 10000 && apt-get install \ + autoconf \ + automake \ + bison \ + build-essential \ + gawk \ + gcc \ + libtool \ + make \ + tar \ + texinfo \ + xz-utils \ + unzip diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 1b47dba4..95ef58b2 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -156,6 +156,7 @@ aarch64-apple-ios: aarch64-unknown-linux-gnu: host_platforms: - linux_x86_64 + - linux_aarch64 pythons_supported: - '3.9' - '3.10' @@ -163,11 +164,16 @@ aarch64-unknown-linux-gnu: - '3.12' - '3.13' - '3.14' - docker_image_suffix: .cross - host_cc: /usr/bin/x86_64-linux-gnu-gcc - host_cxx: /usr/bin/x86_64-linux-gnu-g++ - target_cc: /usr/bin/aarch64-linux-gnu-gcc - target_cxx: /usr/bin/aarch64-linux-gnu-g++ + docker_image_suffix: .debian9 + needs_toolchain: true + host_cc: clang + host_cxx: clang++ + target_cc: clang + target_cxx: clang++ + target_cflags: + - '-fvisibility=hidden' + # Needed to prevent BOLT from crashing. + - '-fdebug-default-version=4' needs: - autoconf - bdb @@ -193,6 +199,11 @@ aarch64-unknown-linux-gnu: - xz - zlib openssl_target: linux-aarch64 + # Blocked on: + # BOLT-ERROR: Cannot relax adr in non-simple function + # trampoline_code_table/1. Use --strict option to override + # See https://github.com/llvm/llvm-project/issues/146541 + # bolt_capable: true arm64-apple-tvos: host_platforms: diff --git a/pythonbuild/docker.py b/pythonbuild/docker.py index 4269b2bd..0be78e4f 100644 --- a/pythonbuild/docker.py +++ b/pythonbuild/docker.py @@ -29,8 +29,10 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path): write_if_different(dest_dir / f, data.encode("utf-8")) -def build_docker_image(client, image_data: bytes, image_dir: pathlib.Path, name): - image_path = image_dir / ("image-%s" % name) +def build_docker_image( + client, image_data: bytes, image_dir: pathlib.Path, name, host_platform +): + image_path = image_dir / f"image-{name}.{host_platform}" return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path) @@ -66,11 +68,14 @@ def ensure_docker_image(client, fh, image_path=None): return image -def get_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name): +def get_image( + client, source_dir: pathlib.Path, image_dir: pathlib.Path, name, host_platform +): if client is None: return None - image_path = image_dir / ("image-%s" % name) + image_name = f"image-{name}.{host_platform}" + image_path = image_dir / image_name tar_path = image_path.with_suffix(".tar") with image_path.open("r") as fh: @@ -88,7 +93,9 @@ def get_image(client, source_dir: pathlib.Path, image_dir: pathlib.Path, name): return image_id else: - return build_docker_image(client, str(source_dir).encode(), image_dir, name) + return build_docker_image( + client, str(source_dir).encode(), image_dir, name, host_platform + ) def copy_file_to_container(path, container, container_path, archive_path=None): diff --git a/pythonbuild/downloads.py b/pythonbuild/downloads.py index c1cae140..24a9917e 100644 --- a/pythonbuild/downloads.py +++ b/pythonbuild/downloads.py @@ -172,6 +172,13 @@ "version": "14.0.3+20220508", }, # Remember to update LLVM_URL in src/release.rs whenever upgrading. + "llvm-20-aarch64-linux": { + "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-aarch64-unknown-linux-gnu.tar.zst", + "size": 255946687, + "sha256": "e70753f294b8f83fffbaf07af36857c27ceaef0291cb10f724ada6af11b0a5bc", + "version": "20.1.4+20250511", + }, + # Remember to update LLVM_URL in src/release.rs whenever upgrading. "llvm-20-x86_64-linux": { "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20250511/llvm-20.1.4+20250511-gnu_only-x86_64-unknown-linux-gnu.tar.zst", "size": 299883811, diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index a8132ca8..0edc11d7 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -39,6 +39,8 @@ def current_host_platform() -> str: if sys.platform == "linux": if machine == "x86_64": return "linux_x86_64" + elif machine == "aarch64": + return "linux_aarch64" else: raise Exception(f"unsupported Linux host platform: {machine}") elif sys.platform == "darwin": @@ -57,6 +59,8 @@ def default_target_triple() -> str: host = current_host_platform() if host == "linux_x86_64": return "x86_64-unknown-linux-gnu" + elif host == "linux_aarch64": + return "aarch64-unknown-linux-gnu" elif host == "macos_arm64": return "aarch64-apple-darwin" elif host == "macos_x86_64": @@ -84,6 +88,8 @@ def supported_targets(yaml_path: pathlib.Path): for host_platform in settings["host_platforms"]: if sys.platform == "linux" and host_platform == "linux_x86_64": targets.add(target) + elif sys.platform == "linux" and host_platform == "linux_aarch64": + targets.add(target) elif sys.platform == "darwin" and host_platform.startswith("macos_"): targets.add(target) @@ -194,7 +200,13 @@ def write_triples_makefiles( image_suffix = settings.get("docker_image_suffix", "") + # On cross builds, we can just use the bare `gcc` image + gcc_image_suffix = ( + image_suffix if not image_suffix.startswith(".cross") else "" + ) + lines.append("DOCKER_IMAGE_BUILD := build%s\n" % image_suffix) + lines.append("DOCKER_IMAGE_GCC := gcc%s\n" % gcc_image_suffix) entry = clang_toolchain(host_platform, triple) lines.append( @@ -470,6 +482,8 @@ def clang_toolchain(host_platform: str, target_triple: str) -> str: return "llvm-14-x86_64-linux" else: return "llvm-20-x86_64-linux" + elif host_platform == "linux_aarch64": + return "llvm-20-aarch64-linux" elif host_platform == "macos_arm64": return "llvm-aarch64-macos" elif host_platform == "macos_x86_64":