diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..3d51051 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,18 @@ +# Run linters on code in repo +# * shellcheck and shfmt for shell scripts +# * various tools for Python scripts +name: Lint checks +on: [push, pull_request] +jobs: + python: + strategy: + fail-fast: false + matrix: + version: ['3.11', '3.10', '3.9', '3.8'] + uses: ClangBuiltLinux/actions-workflows/.github/workflows/python_lint.yml@main + with: + python_version: ${{ matrix.version }} + shellcheck: + uses: ClangBuiltLinux/actions-workflows/.github/workflows/shellcheck.yml@main + shfmt: + uses: ClangBuiltLinux/actions-workflows/.github/workflows/shfmt.yml@main diff --git a/.gitignore b/.gitignore index 375117c..d3211ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /.idea/ +/build/ /venv/ *.pyc -llvm-project/ -build/ diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000..728e3f0 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,3 @@ +[style] +based_on_style = pep8 +column_limit = 100 diff --git a/README.md b/README.md index 4e3cce1..28727c1 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ These scripts have been tested in a Docker image of the following distributions git \ libelf-dev \ libssl-dev \ + libstdc++-$(apt list libstdc++6 2>/dev/null | grep -Eos '[0-9]+\.[0-9]+\.[0-9]+' | head -1 | cut -d . -f 1)-dev \ lld \ make \ ninja-build \ diff --git a/build-binutils.py b/build-binutils.py index 2041e81..9cfe016 100755 --- a/build-binutils.py +++ b/build-binutils.py @@ -1,266 +1,129 @@ #!/usr/bin/env python3 -# Description: Builds a standalone copy of binutils - -import argparse -import multiprocessing -import os -import pathlib -import platform -import shutil -import subprocess -import utils - - -def host_arch_target(): - """ - Converts the host architecture to the first part of a target triple - :return: Target host - """ - host_mapping = { - "armv7l": "arm", - "ppc64": "powerpc64", - "ppc64le": "powerpc64le", - "ppc": "powerpc" - } - machine = platform.machine() - return host_mapping.get(machine, machine) - - -def target_arch(target): - """ - Returns the architecture from a target triple - :param target: Triple to deduce architecture from - :return: Architecture associated with given triple - """ - return target.split("-")[0] - - -def host_is_target(target): - """ - Checks if the current target triple the same as the host. - :param target: Triple to match host architecture against - :return: True if host and target are same, False otherwise - """ - return host_arch_target() == target_arch(target) - - -def parse_parameters(root_folder): - """ - Parses parameters passed to the script into options - :param root_folder: The directory where the script is being invoked from - :return: A 'Namespace' object with all the options parsed from supplied parameters - """ - parser = argparse.ArgumentParser() - parser.add_argument("-B", - "--build-folder", - help=""" - By default, the script will create a "build" folder in the same folder as this script, - then a "binutils" folder within that one and build the files there. If you would like - that done somewhere else, pass it to this parameter. This can either be an absolute - or relative path. - """, - type=str, - default=os.path.join(root_folder.as_posix(), "build", - "binutils")) - parser.add_argument("-I", - "--install-folder", - help=""" - By default, the script will create an "install" folder in the same folder as this script - and install binutils there. If you'd like to have it installed somewhere else, pass - it to this parameter. This can either be an absolute or relative path. - """, - type=str, - default=os.path.join(root_folder.as_posix(), - "install")) - parser.add_argument("-t", - "--targets", - help=""" - The script can build binutils targeting arm-linux-gnueabi, aarch64-linux-gnu, - mipsel-linux-gnu, powerpc-linux-gnu, powerpc64-linux-gnu, powerpc64le-linux-gnu, - riscv64-linux-gnu, s390x-linux-gnu, and x86_64-linux-gnu. - - You can either pass the full target or just the first part (arm, aarch64, x86_64, etc) - or all if you want to build all targets (which is the default). It will only add the - target prefix if it is not for the host architecture. - """, - nargs="+") - parser.add_argument("-m", - "--march", - metavar="ARCH", - help=""" - Add -march=ARCH and -mtune=ARCH to CFLAGS to optimize the toolchain for the target - host processor. - """, - type=str) - return parser.parse_args() - - -def create_targets(targets): - """ - Generate a list of targets that can be passed to the binutils compile function - :param targets: A list of targets to convert to binutils target triples - :return: A list of target triples - """ - targets_dict = { - "arm": "arm-linux-gnueabi", - "aarch64": "aarch64-linux-gnu", - "mips": "mips-linux-gnu", - "mipsel": "mipsel-linux-gnu", - "powerpc64": "powerpc64-linux-gnu", - "powerpc64le": "powerpc64le-linux-gnu", - "powerpc": "powerpc-linux-gnu", - "riscv64": "riscv64-linux-gnu", - "s390x": "s390x-linux-gnu", - "x86_64": "x86_64-linux-gnu" - } - - targets_set = set() - for target in targets: - if target == "all": - return list(targets_dict.values()) - elif target == "host": - key = host_arch_target() - else: - key = target_arch(target) - targets_set.add(targets_dict[key]) - - return list(targets_set) - - -def cleanup(build_folder): - """ - Cleanup the build directory - :param build_folder: Build directory - """ - if build_folder.is_dir(): - shutil.rmtree(build_folder.as_posix()) - build_folder.mkdir(parents=True, exist_ok=True) - - -def invoke_configure(build_folder, install_folder, root_folder, target, - host_arch): - """ - Invokes the configure script to generate a Makefile - :param build_folder: Build directory - :param install_folder: Directory to install binutils to - :param root_folder: Working directory - :param target: Target to compile for - :param host_arch: Host architecture to optimize for - """ - configure = [ - root_folder.joinpath(utils.current_binutils(), "configure").as_posix(), - 'CC=gcc', 'CXX=g++', '--disable-compressed-debug-sections', - '--disable-gdb', '--disable-werror', '--enable-deterministic-archives', - '--enable-new-dtags', '--enable-plugins', '--enable-threads', - '--prefix=%s' % install_folder.as_posix(), '--quiet', - '--with-system-zlib' - ] - if host_arch: - configure += [ - 'CFLAGS=-O2 -march=%s -mtune=%s' % (host_arch, host_arch), - 'CXXFLAGS=-O2 -march=%s -mtune=%s' % (host_arch, host_arch) - ] +# pylint: disable=invalid-name + +from argparse import ArgumentParser +from pathlib import Path +import time + +import tc_build.binutils +import tc_build.utils + +LATEST_BINUTILS_RELEASE = (2, 41, 0) + +parser = ArgumentParser() +parser.add_argument('-B', + '--binutils-folder', + help=''' + By default, the script will download a copy of the binutils source in the src folder within + the same folder as this script. If you have your own copy of the binutils source that you + would like to build from, pass it to this parameter. It can be either an absolute or + relative path. + ''', + type=str) +parser.add_argument('-b', + '--build-folder', + help=''' + By default, the script will create a "build/binutils" folder in the same folder as this + script then build each target in its own folder within that containing folder. If you + would like the containing build folder to be somewhere else, pass it to this parameter. + that done somewhere else, pass it to this parameter. It can be either an absolute or + relative path. + ''', + type=str) +parser.add_argument('-i', + '--install-folder', + help=''' + By default, the script will build binutils but stop before installing it. To install + them into a prefix, pass it to this parameter. This can be either an absolute or + relative path. + ''', + type=str) +parser.add_argument('-m', + '--march', + metavar='ARCH', + help=''' + Add -march=ARCH and -mtune=ARCH to CFLAGS to optimize the toolchain for the target + host processor. + ''', + type=str) +parser.add_argument('--show-build-commands', + help=''' + By default, the script only shows the output of the comands it is running. When this option + is enabled, the invocations of configure and make will be shown to help with reproducing + issues outside of the script. + ''', + action='store_true') +parser.add_argument('-t', + '--targets', + help=''' + The script can build binutils targeting arm-linux-gnueabi, aarch64-linux-gnu, + mips-linux-gnu, mipsel-linux-gnu, powerpc-linux-gnu, powerpc64-linux-gnu, + powerpc64le-linux-gnu, riscv64-linux-gnu, s390x-linux-gnu, and x86_64-linux-gnu. + + By default, it builds all supported targets ("all"). If you would like to build + specific targets only, pass them to this script. It can be either the full target + or just the first part (arm, aarch64, x86_64, etc). + ''', + nargs='+') +args = parser.parse_args() + +script_start = time.time() + +tc_build_folder = Path(__file__).resolve().parent + +bsm = tc_build.binutils.BinutilsSourceManager() +if args.binutils_folder: + bsm.location = Path(args.binutils_folder).resolve() + if not bsm.location.exists(): + raise RuntimeError(f"Provided binutils source ('{bsm.location}') does not exist?") +else: + # Turns (2, 40, 0) into 2.40 and (2, 40, 1) into 2.40.1 to follow tarball names + folder_name = 'binutils-' + '.'.join(str(x) for x in LATEST_BINUTILS_RELEASE if x) + + bsm.location = Path(tc_build_folder, 'src', folder_name) + bsm.tarball.base_download_url = 'https://sourceware.org/pub/binutils/releases' + bsm.tarball.local_location = bsm.location.with_name(f"{folder_name}.tar.xz") + bsm.tarball_remote_checksum_name = 'sha512.sum' + bsm.prepare() + +if args.build_folder: + build_folder = Path(args.build_folder).resolve() +else: + build_folder = Path(tc_build_folder, 'build/binutils') + +default_targets = bsm.default_targets() +if args.targets: + targets = default_targets if 'all' in args.targets else set(args.targets) +else: + targets = default_targets + +targets_to_builder = { + 'arm': tc_build.binutils.ArmBinutilsBuilder, + 'aarch64': tc_build.binutils.AArch64BinutilsBuilder, + 'mips': tc_build.binutils.MipsBinutilsBuilder, + 'mipsel': tc_build.binutils.MipselBinutilsBuilder, + 'powerpc': tc_build.binutils.PowerPCBinutilsBuilder, + 'powerpc64': tc_build.binutils.PowerPC64BinutilsBuilder, + 'powerpc64le': tc_build.binutils.PowerPC64LEBinutilsBuilder, + 'riscv64': tc_build.binutils.RISCV64BinutilsBuilder, + 's390x': tc_build.binutils.S390XBinutilsBuilder, + 'x86_64': tc_build.binutils.X8664BinutilsBuilder, +} +if 'loongarch64' in default_targets: + targets_to_builder['loongarch64'] = tc_build.binutils.LoongArchBinutilsBuilder +for item in targets: + target = item.split('-', maxsplit=1)[0] + if target in targets_to_builder: + builder = targets_to_builder[target]() + builder.folders.build = Path(build_folder, target) + if args.install_folder: + builder.folders.install = Path(args.install_folder).resolve() + builder.folders.source = bsm.location + if args.march: + builder.cflags += [f"-march={args.march}", f"-mtune={args.march}"] + builder.show_commands = args.show_build_commands + builder.build() else: - configure += ['CFLAGS=-O2', 'CXXFLAGS=-O2'] - - configure_arch_flags = { - "arm-linux-gnueabi": [ - '--disable-multilib', '--disable-nls', '--with-gnu-as', - '--with-gnu-ld' - ], - "powerpc-linux-gnu": - ['--disable-sim', '--enable-lto', '--enable-relro', '--with-pic'], - } - configure_arch_flags['aarch64-linux-gnu'] = configure_arch_flags[ - 'arm-linux-gnueabi'] + ['--enable-gold', '--enable-ld=default'] - configure_arch_flags['powerpc64-linux-gnu'] = configure_arch_flags[ - 'powerpc-linux-gnu'] - configure_arch_flags['powerpc64le-linux-gnu'] = configure_arch_flags[ - 'powerpc-linux-gnu'] - configure_arch_flags['riscv64-linux-gnu'] = configure_arch_flags[ - 'powerpc-linux-gnu'] - configure_arch_flags['s390x-linux-gnu'] = configure_arch_flags[ - 'powerpc-linux-gnu'] + ['--enable-targets=s390-linux-gnu'] - configure_arch_flags['x86_64-linux-gnu'] = configure_arch_flags[ - 'powerpc-linux-gnu'] + ['--enable-targets=x86_64-pep'] - - for endian in ["", "el"]: - configure_arch_flags['mips%s-linux-gnu' % (endian)] = [ - '--enable-targets=mips64%s-linux-gnuabi64,mips64%s-linux-gnuabin32' - % (endian, endian) - ] - - configure += configure_arch_flags.get(target, []) - - # If the current machine is not the target, add the prefix to indicate - # that it is a cross compiler - if not host_is_target(target): - configure += ['--program-prefix=%s-' % target, '--target=%s' % target] - - utils.print_header("Building %s binutils" % target) - subprocess.run(configure, check=True, cwd=build_folder.as_posix()) - - -def invoke_make(build_folder, install_folder, target): - """ - Invoke make to compile binutils - :param build_folder: Build directory - :param install_folder: Directory to install binutils to - :param target: Target to compile for - """ - make = ['make', '-s', '-j' + str(multiprocessing.cpu_count()), 'V=0'] - if host_is_target(target): - subprocess.run(make + ['configure-host'], - check=True, - cwd=build_folder.as_posix()) - subprocess.run(make, check=True, cwd=build_folder.as_posix()) - subprocess.run(make + ['prefix=%s' % install_folder.as_posix(), 'install'], - check=True, - cwd=build_folder.as_posix()) - with install_folder.joinpath(".gitignore").open("w") as gitignore: - gitignore.write("*") - - -def build_targets(build, install_folder, root_folder, targets, host_arch): - """ - Builds binutils for all specified targets - :param build: Build directory - :param install_folder: Directory to install binutils to - :param root_folder: Working directory - :param targets: Targets to compile binutils for - :param host_arch: Host architecture to optimize for - :return: - """ - for target in targets: - build_folder = build.joinpath(target) - cleanup(build_folder) - invoke_configure(build_folder, install_folder, root_folder, target, - host_arch) - invoke_make(build_folder, install_folder, target) - - -def main(): - root_folder = pathlib.Path(__file__).resolve().parent - - args = parse_parameters(root_folder) - - build_folder = pathlib.Path(args.build_folder) - if not build_folder.is_absolute(): - build_folder = root_folder.joinpath(build_folder) - - install_folder = pathlib.Path(args.install_folder) - if not install_folder.is_absolute(): - install_folder = root_folder.joinpath(install_folder) - - targets = ["all"] - if args.targets is not None: - targets = args.targets - - utils.download_binutils(root_folder) - - build_targets(build_folder, install_folder, root_folder, - create_targets(targets), args.march) - + tc_build.utils.print_warning(f"Unsupported target ('{target}'), ignoring...") -if __name__ == '__main__': - main() +print(f"\nTotal script duration: {tc_build.utils.get_duration(script_start)}") diff --git a/build-llvm.py b/build-llvm.py index 14816b0..8fe0c1e 100755 --- a/build-llvm.py +++ b/build-llvm.py @@ -1,1614 +1,698 @@ #!/usr/bin/env python3 -# Description: Builds an LLVM toolchain suitable for kernel development +# pylint: disable=invalid-name -import argparse -import datetime -import glob -import pathlib +from argparse import ArgumentParser, RawTextHelpFormatter +from pathlib import Path import platform -import os -import subprocess -import shutil import textwrap import time -import utils -import re -import urllib.request as request -from urllib.error import URLError -# This is a known good revision of LLVM for building the kernel -GOOD_REVISION = '559b8fc17ef6f5a65ccf9a11fce5f91c0a011b00' - - -class Directories: - - def __init__(self, build_folder, install_folder, linux_folder, llvm_folder, - root_folder): - self.build_folder = build_folder - self.install_folder = install_folder - self.linux_folder = linux_folder - self.llvm_folder = llvm_folder - self.root_folder = root_folder - - -class EnvVars: - - def __init__(self, cc, cxx, ld): - self.cc = cc - self.cxx = cxx - self.ld = ld - - -def clang_version(cc, root_folder): - """ - Returns Clang's version as an integer - :param cc: The compiler to check the version of - :param root_folder: Top of the script folder - :return: an int denoting the version of the given compiler - """ - command = [root_folder.joinpath("clang-version.sh").as_posix(), cc] - return int(subprocess.check_output(command).decode()) - - -def parse_parameters(root_folder): - """ - Parses parameters passed to the script into options - :param root_folder: The directory where the script is being invoked from - :return: A 'Namespace' object with all the options parsed from supplied parameters - """ - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter) - clone_options = parser.add_mutually_exclusive_group() - opt_options = parser.add_mutually_exclusive_group() - - parser.add_argument("--assertions", - help=textwrap.dedent("""\ - In a release configuration, assertions are not enabled. Assertions can help catch - issues when compiling but it will increase compile times by 15-20%%. - - """), - action="store_true") - parser.add_argument("-b", - "--branch", - help=textwrap.dedent("""\ - By default, the script builds the main branch (tip of tree) of LLVM. If you would - like to build an older branch, use this parameter. This may be helpful in tracking - down an older bug to properly bisect. This value is just passed along to 'git checkout' - so it can be a branch name, tag name, or hash (unless '--shallow-clone' is used, which - means a hash cannot be used because GitHub does not allow it). - - """), - type=str, - default="main") - parser.add_argument("-B", - "--build-folder", - help=textwrap.dedent("""\ - By default, the script will create a "build" folder in the same folder as this script, - then an "llvm" folder within that one and build the files there. If you would like - that done somewhere else, pass it to this parameter. This can either be an absolute - or relative path. - - """), - type=str, - default=os.path.join(root_folder.as_posix(), "build", - "llvm")) - parser.add_argument("--bolt", - help=textwrap.dedent("""\ - Optimize the final clang binary with BOLT (Binary Optimization and Layout Tool), which can - often improve compile time performance by 5-7%% on average. - - This is similar to Profile Guided Optimization (PGO) but it happens against the final - binary that is built. The script will: - - 1. Figure out if perf can be used with branch sampling. You can test this ahead of time by - running: - - $ perf record --branch-filter any,u --event cycles:u --output /dev/null -- sleep 1 - - 2. If perf cannot be used, the clang binary will be instrumented by llvm-bolt, which will - result in a much slower clang binary. - - NOTE #1: When this instrumentation is combined with a build of LLVM that has already - been PGO'd (i.e., the '--pgo' flag) without LLVM's internal assertions (i.e., - no '--assertions' flag), there might be a crash when attempting to run the - instrumented clang: - - https://github.com/llvm/llvm-project/issues/55004 - - To avoid this, pass '--assertions' with '--bolt --pgo'. - - NOTE #2: BOLT's instrumentation might not be compatible with architectures other than - x86_64 and build-llvm.py's implementation has only been validated on x86_64 - machines: - - https://github.com/llvm/llvm-project/issues/55005 - - BOLT itself only appears to support AArch64 and x86_64 as of LLVM commit - a0b8ab1ba3165d468792cf0032fce274c7d624e1. - - 3. A kernel will be built and profiled. This will either be the host architecture's - defconfig or the first target's defconfig if '--targets' is specified without support - for the host architecture. The profiling data will be quite large, so it is imperative - that you have ample disk space and memory when attempting to do this. With instrumentation, - a profile will be generated for each invocation (PID) of clang, so this data could easily - be a couple hundred gigabytes large. - - 4. The clang binary will be optimized with BOLT using the profile generated above. This can - take some time. - - NOTE #3: Versions of BOLT without commit 7d7771f34d14 ("[BOLT] Compact legacy profiles") - will use significantly more memory during this stage if instrumentation is used - because the merged profile is not as slim as it could be. Either upgrade to a - version of LLVM that contains that change or pick it yourself, switch to perf if - your machine supports it, upgrade the amount of memory you have (if possible), - or run build-llvm.py without '--bolt'. - - """), - action="store_true") - opt_options.add_argument("--build-stage1-only", - help=textwrap.dedent("""\ - By default, the script does a multi-stage build: it builds a more lightweight version of - LLVM first (stage 1) then uses that build to build the full toolchain (stage 2). This - is also known as bootstrapping. - - This option avoids that, building the first stage as if it were the final stage. Note, - this does not install the first stage only toolchain by default to avoid overwritting an - installed mutlt-stage LLVM toolchain; this option is more intended for quick testing - and verification of issues and not regular use. However, if your system is slow or can't - handle 2+ stage builds, you may need this flag. If you would like to install a toolchain - built with this flag, see '--install-stage1-only' below. - - """), - action="store_true") - # yapf: disable - parser.add_argument("--build-type", - metavar='BUILD_TYPE', - help=textwrap.dedent("""\ - By default, the script does a Release build; Debug may be useful for tracking down - particularly nasty bugs. - - See https://llvm.org/docs/GettingStarted.html#compiling-the-llvm-suite-source-code for - more information. - - """), - type=str, - choices=['Release', 'Debug', 'RelWithDebInfo', 'MinSizeRel'], - default="Release") - # yapf: enable - parser.add_argument("--check-targets", - help=textwrap.dedent("""\ - By default, no testing is run on the toolchain. If you would like to run unit/regression - tests, use this parameter to specify a list of check targets to run with ninja. Common - ones include check-llvm, check-clang, and check-lld. - - The values passed to this parameter will be automatically concatenated with 'check-'. - - Example: '--check-targets clang llvm' will make ninja invokve 'check-clang' and 'check-llvm'. - - """), - nargs="+") - parser.add_argument("--clang-vendor", - help=textwrap.dedent("""\ - Add this value to the clang version string (like "Apple clang version..." or - "Android clang version..."). Useful when reverting or applying patches on top - of upstream clang to differentiate a toolchain built with this script from - upstream clang or to distinguish a toolchain built with this script from the - system's clang. Defaults to ClangBuiltLinux, can be set to an empty string to - override this and have no vendor in the version string. - - """), - type=str, - default="ClangBuiltLinux") - parser.add_argument("-D", - "--defines", - help=textwrap.dedent("""\ - Specify additional cmake values. These will be applied to all cmake invocations. - - Example: -D LLVM_PARALLEL_COMPILE_JOBS=2 LLVM_PARALLEL_LINK_JOBS=2 - - See https://llvm.org/docs/CMake.html for various cmake values. Note that some of - the options to this script correspond to cmake values. - - """), - nargs="+") - parser.add_argument("-f", - "--full-toolchain", - help=textwrap.dedent("""\ - By default, the script tunes LLVM for building the Linux kernel by disabling several - projects, targets, and configuration options, which speeds up build times but limits - how the toolchain could be used. - - With this option, all projects and targets are enabled and the script tries to avoid - unnecessarily turning off configuration options. The '--projects' and '--targets' options - to the script can still be used to change the list of projects and targets. This is - useful when using the script to do upstream LLVM development or trying to use LLVM as a - system-wide toolchain. - - """), - action="store_true") - parser.add_argument("-i", - "--incremental", - help=textwrap.dedent("""\ - By default, the script removes all build artifacts from previous compiles. This - prevents that, allowing for dirty builds and faster compiles. - - """), - action="store_true") - parser.add_argument("-I", - "--install-folder", - help=textwrap.dedent("""\ - By default, the script will create an "install" folder in the same folder as this script - and install the LLVM toolchain there. If you'd like to have it installed somewhere - else, pass it to this parameter. This can either be an absolute or relative path. - - """), - type=str, - default=os.path.join(root_folder.as_posix(), - "install")) - parser.add_argument("--install-stage1-only", - help=textwrap.dedent("""\ - When doing a stage 1 only build with '--build-stage1-only', install the toolchain to - the value of INSTALL_FOLDER. - - """), - action="store_true") - parser.add_argument("-l", - "--llvm-folder", - help=textwrap.dedent("""\ - By default, the script will clone the llvm-project into the tc-build repo. If you have - another LLVM checkout that you would like to work out of, pass it to this parameter. - This can either be an absolute or relative path. Implies '--no-update'. - - """), - type=str) - parser.add_argument("-L", - "--linux-folder", - help=textwrap.dedent("""\ - If building with PGO, use this kernel source for building profiles instead of downloading - a tarball from kernel.org. This should be the full or relative path to a complete kernel - source directory, not a tarball or zip file. - - """), - type=str) - parser.add_argument("--lto", - metavar="LTO_TYPE", - help=textwrap.dedent("""\ - Build the final compiler with either ThinLTO (thin) or full LTO (full), which can - often improve compile time performance by 3-5%% on average. - - Only use full LTO if you have more than 64 GB of memory. ThinLTO uses way less memory, - compiles faster because it is fully multithreaded, and it has almost identical - performance (within 1%% usually) to full LTO. The compile time impact of ThinLTO is about - 5x the speed of a '--build-stage1-only' build and 3.5x the speed of a default build. LTO - is much worse and is not worth considering unless you have a server available to build on. - - This option should not be used with '--build-stage1-only' unless you know that your - host compiler and linker support it. See the two links below for more information. - - https://llvm.org/docs/LinkTimeOptimization.html - https://clang.llvm.org/docs/ThinLTO.html - - """), - type=str, - choices=['thin', 'full']) - parser.add_argument("-n", - "--no-update", - help=textwrap.dedent("""\ - By default, the script always updates the LLVM repo before building. This prevents - that, which can be helpful during something like bisecting or manually managing the - repo to pin it to a particular revision. - - """), - action="store_true") - parser.add_argument("--no-ccache", - help=textwrap.dedent("""\ - By default, the script adds LLVM_CCACHE_BUILD to the cmake options so that ccache is - used for the stage one build. This helps speed up compiles but it is only useful for - stage one, which is built using the host compiler, which usually does not change, - resulting in more cache hits. Subsequent stages will be always completely clean builds - since ccache will have no hits due to using a new compiler and it will unnecessarily - fill up the cache with files that will never be called again due to changing compilers - on the next build. This option prevents ccache from being used even at stage one, which - could be useful for benchmarking clean builds. - - """), - action="store_true") - parser.add_argument("-p", - "--projects", - help=textwrap.dedent("""\ - Currently, the script only enables the clang, compiler-rt, lld, and polly folders in LLVM. - If you would like to override this, you can use this parameter and supply a list that is - supported by LLVM_ENABLE_PROJECTS. - - See step #5 here: https://llvm.org/docs/GettingStarted.html#getting-started-quickly-a-summary - - Example: -p \"clang;lld;libcxx\" - - """), - type=str) - opt_options.add_argument("--pgo", - metavar="PGO_BENCHMARK", - help=textwrap.dedent("""\ - Build the final compiler with Profile Guided Optimization, which can often improve compile - time performance by 15-20%% on average. The script will: - - 1. Build a small bootstrap compiler like usual (stage 1). - - 2. Build an instrumented compiler with that compiler (stage 2). - - 3. Run the specified benchmark(s). - - kernel-defconfig, kernel-allmodconfig, kernel-allyesconfig: - - Download and extract kernel source from kernel.org (unless '--linux-folder' is - specified), build the necessary binutils if not found in PATH, and build some - kernels based on the requested config with the instrumented compiler (based on the - '--targets' option). If there is a build error with one of the kernels, build-llvm.py - will fail as well. - - kernel-defconfig-slim, kernel-allmodconfig-slim, kernel-allyesconfig-slim: - - Same as above but only one kernel will be built. If the host architecture is in the list - of targets, that architecture's requested config will be built; otherwise, the config of - the first architecture in '--targets' will be built. This will result in a less optimized - toolchain than the full variant above but it will result in less time spent profiling, - which means less build time overall. This might be worthwhile if you want to take advantage - of PGO on slower machines. - - llvm: - - The script will run the LLVM tests if they were requested via '--check-targets' then - build a full LLVM toolchain with the instrumented compiler. - - 4. Build a final compiler with the profile data generated from step 3 (stage 3). - - Due to the nature of this process, '--build-stage1-only' cannot be used. There will be - three distinct LLVM build folders/compilers and several kernel builds done by default so - ensure that you have enough space on your disk to hold this (25GB should be enough) and the - time/patience to build three toolchains and kernels (will often take 5x the amount of time - as '--build-stage1-only' and 4x the amount of time as the default two-stage build that the - script does). When combined with '--lto', the compile time impact is about 9-10x of a one or - two stage builds. - - See https://llvm.org/docs/HowToBuildWithPGO.html for more information. - - """), - nargs="+", - choices=[ - 'kernel-defconfig', 'kernel-allmodconfig', - 'kernel-allyesconfig', - 'kernel-defconfig-slim', - 'kernel-allmodconfig-slim', - 'kernel-allyesconfig-slim', 'llvm' - ]) - clone_options.add_argument("-s", - "--shallow-clone", - help=textwrap.dedent("""\ - Only fetch the required objects and omit history when cloning the LLVM repo. This - option is only used for the initial clone, not subsequent fetches. This can break - the script's ability to automatically update the repo to newer revisions or branches - so be careful using this. This option is really designed for continuous integration - runs, where a one off clone is necessary. A better option is usually managing the repo - yourself: - - https://github.com/ClangBuiltLinux/tc-build#build-llvmpy - - A couple of notes: - - 1. This cannot be used with '--use-good-revision'. - - 2. When no '--branch' is specified, only main is fetched. To work with other branches, - a branch other than main needs to be specified when the repo is first cloned. - - """), - action="store_true") - parser.add_argument("--show-build-commands", - help=textwrap.dedent("""\ - By default, the script only shows the output of the comands it is running. When this option - is enabled, the invocations of cmake, ninja, and kernel/build.sh will be shown to help with - reproducing issues outside of the script. - - """), - action="store_true") - parser.add_argument("-t", - "--targets", - help=textwrap.dedent("""\ - LLVM is multitargeted by default. Currently, this script only enables the arm32, aarch64, - bpf, mips, powerpc, riscv, s390, and x86 backends because that's what the Linux kernel is - currently concerned with. If you would like to override this, you can use this parameter - and supply a list that is supported by LLVM_TARGETS_TO_BUILD: - - https://llvm.org/docs/CMake.html#llvm-specific-variables - - Example: -t "AArch64;X86" - - """), - type=str) - clone_options.add_argument("--use-good-revision", - help=textwrap.dedent("""\ - By default, the script updates LLVM to the latest tip of tree revision, which may at times be - broken or not work right. With this option, it will checkout a known good revision of LLVM - that builds and works properly. If you use this option often, please remember to update the - script as the known good revision will change. - - NOTE: This option cannot be used with '--shallow-clone'. - - """), - action="store_true") - - return parser.parse_args() - - -def linker_test(cc, ld): - """ - Test to see if the supplied ld value will work with cc -fuse=ld - :param cc: A working C compiler to compile the test program - :param ld: A linker to test -fuse=ld against - :return: 0 if the linker supports -fuse=ld, 1 otherwise - """ - echo = subprocess.Popen(['echo', 'int main() { return 0; }'], - stdout=subprocess.PIPE) - return subprocess.run( - [cc, '-fuse-ld=' + ld, '-o', '/dev/null', '-x', 'c', '-'], - stdin=echo.stdout, - stderr=subprocess.DEVNULL).returncode - - -def versioned_binaries(binary_name): - """ - Returns a list of versioned binaries that may be used on Debian/Ubuntu - :param binary_name: The name of the binary that we're checking for - :return: List of versioned binaries - """ - - # There might be clang-7 to clang-11 - tot_llvm_ver = 11 - try: - response = request.urlopen( - 'https://raw.githubusercontent.com/llvm/llvm-project/main/llvm/CMakeLists.txt' - ) - to_parse = None - data = response.readlines() - for line in data: - line = line.decode('utf-8').strip() - if "set(LLVM_VERSION_MAJOR" in line: - to_parse = line - break - tot_llvm_ver = re.search('\d+', to_parse).group(0) - except URLError: - pass - return [ - '%s-%s' % (binary_name, i) for i in range(int(tot_llvm_ver), 6, -1) - ] - - -def check_cc_ld_variables(root_folder): - """ - Sets the cc, cxx, and ld variables, which will be passed to cmake - :return: A tuple of valid cc, cxx, ld values that can be used to compile LLVM - """ - utils.print_header("Checking CC and LD") - cc, linker, ld = None, None, None - # If the user specified a C compiler, get its full path - if 'CC' in os.environ: - cc = shutil.which(os.environ['CC']) - # Otherwise, try to find one - else: - possible_compilers = versioned_binaries("clang") + ['clang', 'gcc'] - for compiler in possible_compilers: - cc = shutil.which(compiler) - if cc is not None: - break - if cc is None: - raise RuntimeError( - "Neither gcc nor clang could be found on your system!") - - # Evaluate if CC is a symlink. Certain packages of clang (like from - # apt.llvm.org) symlink the clang++ binary to clang++- in - # /usr/bin, which then points to something like /usr/lib/llvm-= 30900: - ld = shutil.which(ld) - if linker_test(cc, ld): - print("LD won't work with " + cc + - ", saving you from yourself by ignoring LD value") - ld = None - # If the user didn't specify a linker - else: - # and we're using clang, try to find the fastest one - if "clang" in cc: - possible_linkers = ['lld', 'gold', 'bfd'] - for linker in possible_linkers: - # We want to find lld wherever the clang we are using is located - ld = shutil.which("ld." + linker, - path=cc_folder + ":" + os.environ['PATH']) - if ld is not None: - break - # If clang is older than 3.9, it won't accept absolute paths so we - # need to just pass it the name (and modify PATH so that it is found properly) - # https://github.com/llvm/llvm-project/commit/e43b7413597d8102a4412f9de41102e55f4f2ec9 - if clang_version(cc, root_folder) < 30900: - os.environ['PATH'] = cc_folder + ":" + os.environ['PATH'] - ld = linker - # and we're using gcc, try to use gold - else: - ld = "gold" - if linker_test(cc, ld): - ld = None - - # Print what binaries we are using to compile/link with so the user can - # decide if that is proper or not - print("CC: " + cc) - print("CXX: " + cxx) - if ld is not None: - ld = ld.strip() - ld_to_print = shutil.which("ld." + ld) - if ld_to_print is None: - ld_to_print = shutil.which(ld) - print("LD: " + ld_to_print) - - return cc, cxx, ld - - -def check_dependencies(): - """ - Makes sure that the base dependencies of cmake, curl, git, and ninja are installed - """ - utils.print_header("Checking dependencies") - required_commands = ["cmake", "curl", "git", "ninja"] - for command in required_commands: - output = shutil.which(command) - if output is None: - raise RuntimeError(command + - " could not be found, please install it!") - print(output) - - -def repo_is_shallow(repo): - """ - Check if repo is a shallow clone already (looks for /.git/shallow) - :param repo: The path to the repo to check - :return: True if the repo is shallow, False if not - """ - git_dir = subprocess.check_output(["git", "rev-parse", "--git-dir"], - cwd=repo.as_posix()).decode().strip() - return pathlib.Path(repo).resolve().joinpath(git_dir, "shallow").exists() - - -def ref_exists(repo, ref): - """ - Check if ref exists using show-branch (works for branches, tags, and raw SHAs) - :param repo: The path to the repo to check - :param ref: The ref to check - :return: True if ref exits, False if not - """ - return subprocess.run(["git", "show-branch", ref], - stderr=subprocess.STDOUT, - stdout=subprocess.DEVNULL, - cwd=repo.as_posix()).returncode == 0 - - -def fetch_llvm_binutils(root_folder, llvm_folder, update, shallow, ref): - """ - Download llvm and binutils or update them if they exist - :param root_folder: Working directory - :param llvm_folder: llvm-project repo directory - :param update: Boolean indicating whether sources need to be updated or not - :param ref: The ref to checkout the monorepo to - """ - cwd = llvm_folder.as_posix() - if llvm_folder.is_dir(): - if update: - utils.print_header("Updating LLVM") - - # Make sure repo is up to date before trying to see if checkout is possible - subprocess.run(["git", "fetch", "origin"], check=True, cwd=cwd) - - # Explain to the user how to avoid issues if their ref does not exist with - # a shallow clone. - if repo_is_shallow(llvm_folder) and not ref_exists( - llvm_folder, ref): - utils.print_error( - "\nSupplied ref (%s) does not exist, cannot checkout." % - ref) - utils.print_error("To proceed, either:") - utils.print_error( - "\t1. Manage the repo yourself and pass '--no-update' to the script." - ) - utils.print_error( - "\t2. Run 'git -C %s fetch --unshallow origin' to get a complete repository." - % cwd) - utils.print_error( - "\t3. Delete '%s' and re-run the script with '-s' + '-b ' to get a full set of refs." - % cwd) - exit(1) - - # Do the update - subprocess.run(["git", "checkout", ref], check=True, cwd=cwd) - local_ref = None - try: - local_ref = subprocess.check_output( - ["git", "symbolic-ref", "-q", "HEAD"], - cwd=cwd).decode("utf-8") - except subprocess.CalledProcessError: - # This is thrown when we're on a revision that cannot be mapped to a symbolic reference, like a tag - # or a git hash. Swallow and move on with the rest of our business. - pass - if local_ref and local_ref.startswith("refs/heads/"): - # This is a branch, pull from remote - subprocess.run([ - "git", "pull", "--rebase", "origin", - local_ref.strip().replace("refs/heads/", "") - ], - check=True, - cwd=cwd) - else: - utils.print_header("Downloading LLVM") - - extra_args = () - if shallow: - extra_args = ("--depth", "1") - if ref != "main": - extra_args += ("--no-single-branch", ) - subprocess.run([ - "git", "clone", *extra_args, - "https://github.com/llvm/llvm-project", - llvm_folder.as_posix() - ], - check=True) - subprocess.run(["git", "checkout", ref], check=True, cwd=cwd) - - # One might wonder why we are downloading binutils in an LLVM build script :) - # We need it for the LLVMgold plugin, which can be used for LTO with ld.gold, - # which at the time of writing this, is how the Google Pixel 3 kernel is built - # and linked. - utils.download_binutils(root_folder) - - -def cleanup(build_folder, incremental): - """ - Clean up and create the build folder - :param build_folder: The build directory - :param incremental: Whether the build is incremental or not. - :return: - """ - if not incremental and build_folder.is_dir(): - shutil.rmtree(build_folder.as_posix()) - build_folder.mkdir(parents=True, exist_ok=True) - - -def get_final_stage(args): - """ - Gets the final stage number, which depends on PGO or a stage one only build - :param args: The args variable generated by parse_parameters - :return: The final stage number - """ - if args.build_stage1_only: - return 1 - elif args.pgo: - return 3 - else: - return 2 - - -def should_install_toolchain(args, stage): - """ - Returns true if the just built toolchain should be installed - :param args: The args variable generated by parse_parameters - :param stage: What stage we are at - :return: True when the toolchain should be installed; see function comments for more details - """ - # We shouldn't install the toolchain if we are not on the final stage - if stage != get_final_stage(args): - return False - - # We shouldn't install the toolchain if the user is only building stage 1 build - # and they didn't explicitly request an install - if args.build_stage1_only and not args.install_stage1_only: - return False - - # Otherwise, we should install the toolchain to the install folder - return True - - -def bootstrap_stage(args, stage): - """ - Returns true if we are doing a multistage build and on stage 1 - :param args: The args variable generated by parse_parameters - :param stage: What stage we are at - :return: True if doing a multistage build and on stage 1, false if not - """ - return not args.build_stage1_only and stage == 1 - - -def instrumented_stage(args, stage): - """ - Returns true if we are using PGO and on stage 2 - :param args: The args variable generated by parse_parameters - :param stage: What stage we are at - :return: True if using PGO and on stage 2, false if not - """ - return args.pgo and stage == 2 - - -def pgo_stage(stage): - """ - Returns true if LLVM is being built as a PGO benchmark - :return: True if LLVM is being built as a PGO benchmark, false if not - """ - return stage == "pgo" - - -def slim_cmake_defines(): - """ - Generate a set of cmake defines to slim down the LLVM toolchain - :return: A set of defines - """ - # yapf: disable - defines = { - # Objective-C Automatic Reference Counting (we don't use Objective-C) - # https://clang.llvm.org/docs/AutomaticReferenceCounting.html - 'CLANG_ENABLE_ARCMT': 'OFF', - # We don't (currently) use the static analyzer and it saves cycles - # according to Chromium OS: - # https://crrev.com/44702077cc9b5185fc21e99485ee4f0507722f82 - 'CLANG_ENABLE_STATIC_ANALYZER': 'OFF', - # We don't use the plugin system and it will remove unused symbols: - # https://crbug.com/917404 - 'CLANG_PLUGIN_SUPPORT': 'OFF', - # Don't build bindings; they are for other languages that the kernel does not use - 'LLVM_ENABLE_BINDINGS': 'OFF', - # Don't build Ocaml documentation - 'LLVM_ENABLE_OCAMLDOC': 'OFF', - # Don't build clang-tools-extras to cut down on build targets (about 400 files or so) - 'LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR': '', - # Don't include documentation build targets because it is available on the web - 'LLVM_INCLUDE_DOCS': 'OFF', - # Don't include example build targets to save on cmake cycles - 'LLVM_INCLUDE_EXAMPLES': 'OFF' - } - # yapf: enable - - return defines - - -def get_stage_binary(binary, dirs, stage): - """ - Generate a path from the stage bin directory for the requested binary - :param binary: Name of the binary - :param dirs: An instance of the Directories class with the paths to use - :param stage: The staged binary to use - :return: A path suitable for a cmake define - """ - return dirs.build_folder.joinpath("stage%d" % stage, "bin", - binary).as_posix() - - -def if_binary_exists(binary_name, cc): - """ - Returns the path of the requested binary if it exists and clang is being used, None if not - :param binary_name: Name of the binary - :param cc: Path to CC binary - :return: A path to binary if it exists and clang is being used, None if either condition is false - """ - binary = None - if "clang" in cc: - binary = shutil.which(binary_name, - path=os.path.dirname(cc) + ":" + - os.environ['PATH']) - return binary - - -def cc_ld_cmake_defines(dirs, env_vars, stage): - """ - Generate compiler and linker cmake defines, which change depending on what - stage we are at - :param dirs: An instance of the Directories class with the paths to use - :param env_vars: An instance of the EnvVars class with the compilers/linker to use - :param stage: What stage we are at - :return: A set of defines - """ - defines = {} - - if stage == 1: - # Already figured out above - cc = env_vars.cc - cxx = env_vars.cxx - ld = env_vars.ld - # Optional to have - ar = if_binary_exists("llvm-ar", cc) - ranlib = if_binary_exists("llvm-ranlib", cc) - # Cannot be used from host due to potential incompatibilities - clang_tblgen = None - llvm_tblgen = None - else: - if pgo_stage(stage): - stage = 2 - else: - stage = 1 - ar = get_stage_binary("llvm-ar", dirs, stage) - cc = get_stage_binary("clang", dirs, stage) - clang_tblgen = get_stage_binary("clang-tblgen", dirs, stage) - cxx = get_stage_binary("clang++", dirs, stage) - ld = get_stage_binary("ld.lld", dirs, stage) - llvm_tblgen = get_stage_binary("llvm-tblgen", dirs, stage) - ranlib = get_stage_binary("llvm-ranlib", dirs, stage) - - # Use llvm-ar for stage 2+ builds to avoid errors with bfd plugin - # bfd plugin: LLVM gold plugin has failed to create LTO module: Unknown attribute kind (60) (Producer: 'LLVM9.0.0svn' Reader: 'LLVM 8.0.0') - if ar: - defines['CMAKE_AR'] = ar - - # The C compiler to use - defines['CMAKE_C_COMPILER'] = cc - - if clang_tblgen: - defines['CLANG_TABLEGEN'] = clang_tblgen - - # The C++ compiler to use - defines['CMAKE_CXX_COMPILER'] = cxx - - # If we have a linker, use it - if ld: - defines['LLVM_USE_LINKER'] = ld - - if llvm_tblgen: - defines['LLVM_TABLEGEN'] = llvm_tblgen - - # Use llvm-ranlib for stage 2+ builds - if ranlib: - defines['CMAKE_RANLIB'] = ranlib - - return defines - - -def distro_cmake_defines(): - """ - Generate distribution specific cmake defines - :return: A set of defines - """ - defines = {} - - # Clear Linux needs a different target to find all of the C++ header files, otherwise - # stage 2+ compiles will fail without this - # We figure this out based on the existence of x86_64-generic-linux in the C++ headers path - if glob.glob("/usr/include/c++/*/x86_64-generic-linux"): - defines['LLVM_HOST_TRIPLE'] = "x86_64-generic-linux" - - # By default, the Linux triples are for glibc, which might not work on - # musl-based systems. If clang is available, get the default target triple - # from it so that clang without a '--target' flag always works. - if shutil.which("clang"): - clang_cmd = ["clang", "-print-target-triple"] - clang_cmd_out = subprocess.run(clang_cmd, - capture_output=True, - check=True) - default_target_triple = clang_cmd_out.stdout.decode('UTF-8').strip() - defines['LLVM_DEFAULT_TARGET_TRIPLE'] = default_target_triple - - return defines - - -def project_cmake_defines(args, stage): - """ - Generate lists of projects, depending on whether a full or - kernel-focused LLVM build is being done and the stage - :param args: The args variable generated by parse_parameters - :param stage: What stage we are at - :return: A set of defines - """ - defines = {} +import tc_build.utils - if args.full_toolchain: - if args.projects: - projects = args.projects - else: - projects = "all" - else: - if bootstrap_stage(args, stage): - projects = "clang;lld" - if args.bolt: - projects += ';bolt' - if args.pgo: - projects += ';compiler-rt' - elif instrumented_stage(args, stage): - projects = "clang;lld" - elif args.projects: - projects = args.projects - else: - projects = "clang;compiler-rt;lld;polly" - - # Add "bolt" in the list of projects if the user is doing a one stage build - # and it is not already in the list - if args.bolt and args.build_stage1_only and projects != "all" and "bolt" not in projects: - projects += ";bolt" - - defines['LLVM_ENABLE_PROJECTS'] = projects - - if "compiler-rt" in projects: - if not args.full_toolchain: - # Don't build libfuzzer when compiler-rt is enabled, it invokes cmake again and we don't use it - defines['COMPILER_RT_BUILD_LIBFUZZER'] = 'OFF' - # We only use compiler-rt for the sanitizers, disable some extra stuff we don't need - # Chromium OS also does this: https://crrev.com/c/1629950 - defines['COMPILER_RT_BUILD_CRT'] = 'OFF' - defines['COMPILER_RT_BUILD_XRAY'] = 'OFF' - # We don't need the sanitizers for the stage 1 bootstrap - if bootstrap_stage(args, stage): - defines['COMPILER_RT_BUILD_SANITIZERS'] = 'OFF' - - return defines - - -def get_targets(args): - """ - Gets the list of targets for cmake and kernel/build.sh - :param args: The args variable generated by parse_parameters - :return: A string of targets suitable for cmake or kernel/build.sh - """ - if args.targets: - targets = args.targets - elif args.full_toolchain: - targets = "all" - else: - targets = "AArch64;ARM;BPF;Hexagon;Mips;PowerPC;RISCV;SystemZ;X86" - - return targets +from tc_build.llvm import LLVMBootstrapBuilder, LLVMBuilder, LLVMInstrumentedBuilder, LLVMSlimBuilder, LLVMSlimInstrumentedBuilder, LLVMSourceManager +from tc_build.kernel import KernelBuilder, LinuxSourceManager, LLVMKernelBuilder +from tc_build.tools import HostTools, StageTools +# This is a known good revision of LLVM for building the kernel +GOOD_REVISION = 'b5983a38cbf4eb405fe9583ab89e15db1dcfa173' + +# The version of the Linux kernel that the script downloads if necessary +DEFAULT_KERNEL_FOR_PGO = (6, 4, 0) + +parser = ArgumentParser(formatter_class=RawTextHelpFormatter) +clone_options = parser.add_mutually_exclusive_group() +opt_options = parser.add_mutually_exclusive_group() + +parser.add_argument('--assertions', + help=textwrap.dedent('''\ + In a release configuration, assertions are not enabled. Assertions can help catch + issues when compiling but it will increase compile times by 15-20%%. + + '''), + action='store_true') +parser.add_argument('-b', + '--build-folder', + help=textwrap.dedent('''\ + By default, the script will create a "build/llvm" folder in the same folder as this + script and build each requested stage within that containing folder. To change the + location of the containing build folder, pass it to this parameter. This can be either + an absolute or relative path. + + '''), + type=str) +parser.add_argument('--bolt', + help=textwrap.dedent('''\ + Optimize the final clang binary with BOLT (Binary Optimization and Layout Tool), which can + often improve compile time performance by 5-7%% on average. + + This is similar to Profile Guided Optimization (PGO) but it happens against the final + binary that is built. The script will: + + 1. Figure out if perf can be used with branch sampling. You can test this ahead of time by + running: + + $ perf record --branch-filter any,u --event cycles:u --output /dev/null -- sleep 1 + + 2. If perf cannot be used, the clang binary will be instrumented by llvm-bolt, which will + result in a much slower clang binary. + + NOTE #1: When this instrumentation is combined with a build of LLVM that has already + been PGO'd (i.e., the '--pgo' flag) without LLVM's internal assertions (i.e., + no '--assertions' flag), there might be a crash when attempting to run the + instrumented clang: + https://github.com/llvm/llvm-project/issues/55004 + To avoid this, pass '--assertions' with '--bolt --pgo'. + + NOTE #2: BOLT's instrumentation might not be compatible with architectures other than + x86_64 and build-llvm.py's implementation has only been validated on x86_64 + machines: + https://github.com/llvm/llvm-project/issues/55005 + BOLT itself only appears to support AArch64 and x86_64 as of LLVM commit + a0b8ab1ba3165d468792cf0032fce274c7d624e1. + + 3. A kernel will be built and profiled. This will either be the host architecture's + defconfig or the first target's defconfig if '--targets' is specified without support + for the host architecture. The profiling data will be quite large, so it is imperative + that you have ample disk space and memory when attempting to do this. With instrumentation, + a profile will be generated for each invocation (PID) of clang, so this data could easily + be a couple hundred gigabytes large. + + 4. The clang binary will be optimized with BOLT using the profile generated above. This can + take some time. + + NOTE #3: Versions of BOLT without commit 7d7771f34d14 ("[BOLT] Compact legacy profiles") + will use significantly more memory during this stage if instrumentation is used + because the merged profile is not as slim as it could be. Either upgrade to a + version of LLVM that contains that change or pick it yourself, switch to perf if + your machine supports it, upgrade the amount of memory you have (if possible), + or run build-llvm.py without '--bolt'. + + '''), + action='store_true') +opt_options.add_argument('--build-stage1-only', + help=textwrap.dedent('''\ + By default, the script does a multi-stage build: it builds a more lightweight version of + LLVM first (stage 1) then uses that build to build the full toolchain (stage 2). This + is also known as bootstrapping. + + This option avoids that, building the first stage as if it were the final stage. Note, + this option is more intended for quick testing and verification of issues and not regular + use. However, if your system is slow or can't handle 2+ stage builds, you may need this flag. + + '''), + action='store_true') +# yapf: disable +parser.add_argument('--build-type', + metavar='BUILD_TYPE', + help=textwrap.dedent('''\ + By default, the script does a Release build; Debug may be useful for tracking down + particularly nasty bugs. + + See https://llvm.org/docs/GettingStarted.html#compiling-the-llvm-suite-source-code for + more information. + + '''), + type=str, + choices=['Release', 'Debug', 'RelWithDebInfo', 'MinSizeRel']) +# yapf: enable +parser.add_argument('--check-targets', + help=textwrap.dedent('''\ + By default, no testing is run on the toolchain. If you would like to run unit/regression + tests, use this parameter to specify a list of check targets to run with ninja. Common + ones include check-llvm, check-clang, and check-lld. + + The values passed to this parameter will be automatically concatenated with 'check-'. + + Example: '--check-targets clang llvm' will make ninja invokve 'check-clang' and 'check-llvm'. + + '''), + nargs='+') +parser.add_argument('-D', + '--defines', + help=textwrap.dedent('''\ + Specify additional cmake values. These will be applied to all cmake invocations. + + Example: -D LLVM_PARALLEL_COMPILE_JOBS=2 LLVM_PARALLEL_LINK_JOBS=2 + + See https://llvm.org/docs/CMake.html for various cmake values. Note that some of + the options to this script correspond to cmake values. + + '''), + nargs='+') +parser.add_argument('-f', + '--full-toolchain', + help=textwrap.dedent('''\ + By default, the script tunes LLVM for building the Linux kernel by disabling several + projects, targets, and configuration options, which speeds up build times but limits + how the toolchain could be used. + + With this option, all projects and targets are enabled and the script tries to avoid + unnecessarily turning off configuration options. The '--projects' and '--targets' options + to the script can still be used to change the list of projects and targets. This is + useful when using the script to do upstream LLVM development or trying to use LLVM as a + system-wide toolchain. + + '''), + action='store_true') +parser.add_argument('-i', + '--install-folder', + help=textwrap.dedent('''\ + By default, the script will leave the toolchain in its build folder. To install it + outside the build folder for persistent use, pass the installation location that you + desire to this parameter. This can be either an absolute or relative path. + + '''), + type=str) +parser.add_argument('--install-targets', + help=textwrap.dedent('''\ + By default, the script will just run the 'install' target to install the toolchain to + the desired prefix. To produce a slimmer toolchain, specify the desired targets to + install using this options. + + The values passed to this parameter will be automatically prepended with 'install-'. + + Example: '--install-targets clang lld' will make ninja invoke 'install-clang' and + 'install-lld'. + + '''), + nargs='+') +parser.add_argument('-l', + '--llvm-folder', + help=textwrap.dedent('''\ + By default, the script will clone the llvm-project into the tc-build repo. If you have + another LLVM checkout that you would like to work out of, pass it to this parameter. + This can either be an absolute or relative path. Implies '--no-update'. When this + option is supplied, '--ref' and '--use-good-revison' do nothing, as the script does + not manipulate a repository it does not own. + + '''), + type=str) +parser.add_argument('-L', + '--linux-folder', + help=textwrap.dedent('''\ + If building with PGO, use this kernel source for building profiles instead of downloading + a tarball from kernel.org. This should be the full or relative path to a complete kernel + source directory, not a tarball or zip file. + + '''), + type=str) +parser.add_argument('--lto', + metavar='LTO_TYPE', + help=textwrap.dedent('''\ + Build the final compiler with either ThinLTO (thin) or full LTO (full), which can + often improve compile time performance by 3-5%% on average. + + Only use full LTO if you have more than 64 GB of memory. ThinLTO uses way less memory, + compiles faster because it is fully multithreaded, and it has almost identical + performance (within 1%% usually) to full LTO. The compile time impact of ThinLTO is about + 5x the speed of a '--build-stage1-only' build and 3.5x the speed of a default build. LTO + is much worse and is not worth considering unless you have a server available to build on. + + This option should not be used with '--build-stage1-only' unless you know that your + host compiler and linker support it. See the two links below for more information. + + https://llvm.org/docs/LinkTimeOptimization.html + https://clang.llvm.org/docs/ThinLTO.html + + '''), + type=str, + choices=['thin', 'full']) +parser.add_argument('-n', + '--no-update', + help=textwrap.dedent('''\ + By default, the script always updates the LLVM repo before building. This prevents + that, which can be helpful during something like bisecting or manually managing the + repo to pin it to a particular revision. + + '''), + action='store_true') +parser.add_argument('--no-ccache', + help=textwrap.dedent('''\ + By default, the script adds LLVM_CCACHE_BUILD to the cmake options so that ccache is + used for the stage one build. This helps speed up compiles but it is only useful for + stage one, which is built using the host compiler, which usually does not change, + resulting in more cache hits. Subsequent stages will be always completely clean builds + since ccache will have no hits due to using a new compiler and it will unnecessarily + fill up the cache with files that will never be called again due to changing compilers + on the next build. This option prevents ccache from being used even at stage one, which + could be useful for benchmarking clean builds. + + '''), + action='store_true') +parser.add_argument('-p', + '--projects', + help=textwrap.dedent('''\ + Currently, the script only enables the clang, compiler-rt, lld, and polly folders in LLVM. + If you would like to override this, you can use this parameter and supply a list that is + supported by LLVM_ENABLE_PROJECTS. + + See step #5 here: https://llvm.org/docs/GettingStarted.html#getting-started-quickly-a-summary + + Example: -p clang lld polly + + '''), + nargs='+') +opt_options.add_argument('--pgo', + metavar='PGO_BENCHMARK', + help=textwrap.dedent('''\ + Build the final compiler with Profile Guided Optimization, which can often improve compile + time performance by 15-20%% on average. The script will: + + 1. Build a small bootstrap compiler like usual (stage 1). + + 2. Build an instrumented compiler with that compiler (stage 2). + + 3. Run the specified benchmark(s). + + kernel-defconfig, kernel-allmodconfig, kernel-allyesconfig: + + Download and extract kernel source from kernel.org (unless '--linux-folder' is + specified) and build some kernels based on the requested config with the instrumented + compiler (based on the '--targets' option). If there is a build error with one of the + kernels, build-llvm.py will fail as well. + + kernel-defconfig-slim, kernel-allmodconfig-slim, kernel-allyesconfig-slim: + + Same as above but only one kernel will be built. If the host architecture is in the list + of targets, that architecture's requested config will be built; otherwise, the config of + the first architecture in '--targets' will be built. This will result in a less optimized + toolchain than the full variant above but it will result in less time spent profiling, + which means less build time overall. This might be worthwhile if you want to take advantage + of PGO on slower machines. + + llvm: + + The script will run the LLVM tests if they were requested via '--check-targets' then + build a full LLVM toolchain with the instrumented compiler. + + 4. Build a final compiler with the profile data generated from step 3 (stage 3). + + Due to the nature of this process, '--build-stage1-only' cannot be used. There will be + three distinct LLVM build folders/compilers and several kernel builds done by default so + ensure that you have enough space on your disk to hold this (25GB should be enough) and the + time/patience to build three toolchains and kernels (will often take 5x the amount of time + as '--build-stage1-only' and 4x the amount of time as the default two-stage build that the + script does). When combined with '--lto', the compile time impact is about 9-10x of a one or + two stage builds. + + See https://llvm.org/docs/HowToBuildWithPGO.html for more information. + + '''), + nargs='+', + choices=[ + 'kernel-defconfig', + 'kernel-allmodconfig', + 'kernel-allyesconfig', + 'kernel-defconfig-slim', + 'kernel-allmodconfig-slim', + 'kernel-allyesconfig-slim', + 'llvm', + ]) +parser.add_argument('--quiet-cmake', + help=textwrap.dedent('''\ + By default, the script shows all output from cmake. When this option is enabled, the + invocations of cmake will only show warnings and errors. + + '''), + action='store_true') +parser.add_argument('-r', + '--ref', + help=textwrap.dedent('''\ + By default, the script builds the main branch (tip of tree) of LLVM. If you would + like to build an older branch, use this parameter. This may be helpful in tracking + down an older bug to properly bisect. This value is just passed along to 'git checkout' + so it can be a branch name, tag name, or hash (unless '--shallow-clone' is used, which + means a hash cannot be used because GitHub does not allow it). This will have no effect + if '--llvm-folder' is provided, as the script does not manipulate a repository that it + does not own. + + '''), + default='main', + type=str) +clone_options.add_argument('-s', + '--shallow-clone', + help=textwrap.dedent('''\ + Only fetch the required objects and omit history when cloning the LLVM repo. This + option is only used for the initial clone, not subsequent fetches. This can break + the script's ability to automatically update the repo to newer revisions or branches + so be careful using this. This option is really designed for continuous integration + runs, where a one off clone is necessary. A better option is usually managing the repo + yourself: + + https://github.com/ClangBuiltLinux/tc-build#build-llvmpy + + A couple of notes: + + 1. This cannot be used with '--use-good-revision'. + + 2. When no '--branch' is specified, only main is fetched. To work with other branches, + a branch other than main needs to be specified when the repo is first cloned. + + '''), + action='store_true') +parser.add_argument('--show-build-commands', + help=textwrap.dedent('''\ + By default, the script only shows the output of the comands it is running. When this option + is enabled, the invocations of cmake, ninja, and make will be shown to help with + reproducing issues outside of the script. + + '''), + action='store_true') +parser.add_argument('-t', + '--targets', + help=textwrap.dedent('''\ + LLVM is multitargeted by default. Currently, this script only enables the arm32, aarch64, + bpf, mips, powerpc, riscv, s390, and x86 backends because that's what the Linux kernel is + currently concerned with. If you would like to override this, you can use this parameter + and supply a list of targets supported by LLVM_TARGETS_TO_BUILD: + + https://llvm.org/docs/CMake.html#llvm-specific-variables + + Example: -t AArch64 ARM X86 + + '''), + nargs='+') +clone_options.add_argument('--use-good-revision', + help=textwrap.dedent('''\ + By default, the script updates LLVM to the latest tip of tree revision, which may at times be + broken or not work right. With this option, it will checkout a known good revision of LLVM + that builds and works properly. If you use this option often, please remember to update the + script as the known good revision will change. + + NOTE: This option cannot be used with '--shallow-clone'. + + '''), + action='store_const', + const=GOOD_REVISION, + dest='ref') +parser.add_argument('--vendor-string', + help=textwrap.dedent('''\ + Add this value to the clang and ld.lld version string (like "Apple clang version..." + or "Android clang version..."). Useful when reverting or applying patches on top + of upstream clang to differentiate a toolchain built with this script from + upstream clang or to distinguish a toolchain built with this script from the + system's clang. Defaults to ClangBuiltLinux, can be set to an empty string to + override this and have no vendor in the version string. + + '''), + type=str, + default='ClangBuiltLinux') +args = parser.parse_args() + +# Start tracking time that the script takes +script_start = time.time() + +# Folder validation +tc_build_folder = Path(__file__).resolve().parent +src_folder = Path(tc_build_folder, 'src') + +if args.build_folder: + build_folder = Path(args.build_folder).resolve() +else: + build_folder = Path(tc_build_folder, 'build/llvm') + +# Validate and prepare Linux source if doing BOLT or PGO with kernel benchmarks +# Check for issues early, as these technologies are time consuming, so a user +# might step away from the build once it looks like it has started +if args.bolt or (args.pgo and [x for x in args.pgo if 'kernel' in x]): + lsm = LinuxSourceManager() + if args.linux_folder: + if not (linux_folder := Path(args.linux_folder).resolve()).exists(): + raise RuntimeError(f"Provided Linux folder ('{args.linux_folder}') does not exist?") + if not Path(linux_folder, 'Makefile').exists(): + raise RuntimeError( + f"Provided Linux folder ('{args.linux_folder}') does not appear to be a Linux kernel tree?" + ) -def target_cmake_defines(args, stage): - """ - Generate target cmake define, which change depending on what - stage we are at - :param args: The args variable generated by parse_parameters - :param stage: What stage we are at - :return: A set of defines - """ - defines = {} + lsm.location = linux_folder - if bootstrap_stage(args, stage): - targets = "host" - else: - targets = get_targets(args) - - defines['LLVM_TARGETS_TO_BUILD'] = targets - - return defines - - -def stage_specific_cmake_defines(args, dirs, stage): - """ - Generate other stage specific defines - :param args: The args variable generated by parse_parameters - :param dirs: An instance of the Directories class with the paths to use - :param stage: What stage we are at - :return: A set of defines - """ - defines = {} - - # Use ccache for the stage 1 build as it will usually be done with a consistent - # compiler and won't need a full rebuild very often - if stage == 1 and not args.no_ccache and shutil.which("ccache"): - defines['LLVM_CCACHE_BUILD'] = 'ON' - - if bootstrap_stage(args, stage): - # Based on clang/cmake/caches/Apple-stage1.cmake - defines.update(slim_cmake_defines()) - defines['CMAKE_BUILD_TYPE'] = 'Release' - defines['LLVM_BUILD_UTILS'] = 'OFF' - defines['LLVM_ENABLE_BACKTRACES'] = 'OFF' - defines['LLVM_ENABLE_WARNINGS'] = 'OFF' - defines['LLVM_INCLUDE_TESTS'] = 'OFF' + # The kernel builder used by PGO below is written with a minimum + # version in mind. If the user supplied their own Linux source, make + # sure it is recent enough that the kernel builder will work. + if (linux_version := lsm.get_version()) < KernelBuilder.MINIMUM_SUPPORTED_VERSION: + found_version = '.'.join(map(str, linux_version)) + minimum_version = '.'.join(map(str, KernelBuilder.MINIMUM_SUPPORTED_VERSION)) + raise RuntimeError( + f"Supplied kernel source version ('{found_version}') is older than the minimum required version ('{minimum_version}'), provide a newer version!" + ) else: - # https://llvm.org/docs/CMake.html#frequently-used-cmake-variables - defines['CMAKE_BUILD_TYPE'] = args.build_type - - # We don't care about warnings if we are building a release build - if args.build_type == "Release": - defines['LLVM_ENABLE_WARNINGS'] = 'OFF' - - # Build with assertions enabled if requested (will slow down compilation - # so it is not on by default) - if args.assertions: - defines['LLVM_ENABLE_ASSERTIONS'] = 'ON' - - # Where the toolchain should be installed - defines['CMAKE_INSTALL_PREFIX'] = dirs.install_folder.as_posix() - - # Build with instrumentation if we are using PGO and on stage 2 - if instrumented_stage(args, stage): - defines['LLVM_BUILD_INSTRUMENTED'] = 'IR' - defines['LLVM_BUILD_RUNTIME'] = 'OFF' - # The next two defines is needed to avoid thousands of warnings - # along the lines of: - # "Unable to track new values: Running out of static counters." - defines['LLVM_LINK_LLVM_DYLIB'] = 'ON' - defines['LLVM_VP_COUNTERS_PER_SITE'] = '6' - - # If we are at the final stage, use PGO/Thin LTO if requested - if stage == get_final_stage(args): - if args.pgo: - defines['LLVM_PROFDATA_FILE'] = dirs.build_folder.joinpath( - "profdata.prof").as_posix() - if args.lto: - defines['LLVM_ENABLE_LTO'] = args.lto.capitalize() - # BOLT needs relocations for instrumentation - if args.bolt: - defines['CMAKE_EXE_LINKER_FLAGS'] = '-Wl,--emit-relocs' - - # If the user did not specify CMAKE_C_FLAGS or CMAKE_CXX_FLAGS, add them as empty - # to paste stage 2 to ensure there are no environment issues (since CFLAGS and CXXFLAGS - # are taken into account by cmake) - keys = ['CMAKE_C_FLAGS', 'CMAKE_CXX_FLAGS'] - for key in keys: - if not key in str(args.defines): - defines[key] = '' - - # For LLVMgold.so, which is used for LTO with ld.gold - defines['LLVM_BINUTILS_INCDIR'] = dirs.root_folder.joinpath( - utils.current_binutils(), "include").as_posix() - defines['LLVM_ENABLE_PLUGINS'] = 'ON' - - return defines - - -def build_cmake_defines(args, dirs, env_vars, stage): - """ - Generate cmake defines - :param args: The args variable generated by parse_parameters - :param dirs: An instance of the Directories class with the paths to use - :param env_vars: An instance of the EnvVars class with the compilers/linker to use - :param stage: What stage we are at - :return: A set of defines - """ - defines = {} - - # Get slim defines if we are not building a full toolchain - if not args.full_toolchain: - defines.update(slim_cmake_defines()) - - # Add compiler/linker defines, which change based on stage - defines.update(cc_ld_cmake_defines(dirs, env_vars, stage)) - - # Add distribution specific defines - defines.update(distro_cmake_defines()) - - # Add project and target defines, which change based on stage - defines.update(project_cmake_defines(args, stage)) - defines.update(target_cmake_defines(args, stage)) - - # Add other stage specific defines - defines.update(stage_specific_cmake_defines(args, dirs, stage)) - - # Add the vendor string if necessary - if args.clang_vendor: - defines['CLANG_VENDOR'] = args.clang_vendor - - # Removes system dependency on terminfo to keep the dynamic library dependencies slim - defines['LLVM_ENABLE_TERMINFO'] = 'OFF' - - return defines - - -def show_command(args, command): - """ - :param args: The args variable generated by parse_parameters - :param command: The command being run - """ - if args.show_build_commands: - print("$ %s" % " ".join([str(element) for element in command])) - - -def get_pgo_header_folder(stage): - if pgo_stage(stage): - header_string = "for PGO" - sub_folder = "pgo" + # Turns (6, 2, 0) into 6.2 and (6, 2, 1) into 6.2.1 to follow tarball names + ver_str = '.'.join(str(x) for x in DEFAULT_KERNEL_FOR_PGO if x) + lsm.location = Path(src_folder, f"linux-{ver_str}") + lsm.patches = list(src_folder.glob('*.patch')) + + lsm.tarball.base_download_url = 'https://cdn.kernel.org/pub/linux/kernel/v6.x' + lsm.tarball.local_location = lsm.location.with_name(f"{lsm.location.name}.tar.xz") + lsm.tarball.remote_checksum_name = 'sha256sums.asc' + + tc_build.utils.print_header('Preparing Linux source for profiling runs') + lsm.prepare() + +# Validate and configure LLVM source +if args.llvm_folder: + if not (llvm_folder := Path(args.llvm_folder).resolve()).exists(): + raise RuntimeError(f"Provided LLVM folder ('{args.llvm_folder}') does not exist?") +else: + llvm_folder = Path(src_folder, 'llvm-project') +llvm_source = LLVMSourceManager(llvm_folder) +llvm_source.download(args.ref, args.shallow_clone) +if not (args.llvm_folder or args.no_update): + llvm_source.update(args.ref) + +# Get host tools +tc_build.utils.print_header('Checking CC and LD') + +host_tools = HostTools() +host_tools.show_compiler_linker() + +# '--full-toolchain' affects all stages aside from the bootstrap stage so cache +# the class for all future initializations. +def_llvm_builder_cls = LLVMBuilder if args.full_toolchain else LLVMSlimBuilder + +# Instantiate final builder to validate user supplied targets ahead of time, so +# that the user can correct the issue sooner rather than later. +final = def_llvm_builder_cls() +final.folders.source = llvm_folder +if args.targets: + final.targets = args.targets + final.validate_targets() +else: + final.targets = ['all'] if args.full_toolchain else llvm_source.default_targets() + +# Configure projects +if args.projects: + final.projects = args.projects +elif args.full_toolchain: + final.projects = ['all'] +else: + final.projects = llvm_source.default_projects() + +# Warn the user of certain issues with BOLT and instrumentation +if args.bolt and not final.can_use_perf(): + warned = False + has_4f158995b9cddae = Path(llvm_folder, 'bolt/lib/Passes/ValidateMemRefs.cpp').exists() + if args.pgo and not args.assertions and not has_4f158995b9cddae: + tc_build.utils.print_warning( + 'Using BOLT in instrumentation mode with PGO and no assertions might result in a binary that crashes:' + ) + tc_build.utils.print_warning('https://github.com/llvm/llvm-project/issues/55004') + tc_build.utils.print_warning( + "Consider adding '--assertions' if there are any failures during the BOLT stage.") + warned = True + if platform.machine() != 'x86_64': + tc_build.utils.print_warning( + 'Using BOLT in instrumentation mode may not work on non-x86_64 machines:') + tc_build.utils.print_warning('https://github.com/llvm/llvm-project/issues/55005') + tc_build.utils.print_warning( + "Consider dropping '--bolt' if there are any failures during the BOLT stage.") + warned = True + if warned: + tc_build.utils.print_warning('Continuing in 5 seconds, hit Ctrl-C to cancel...') + time.sleep(5) + +# Figure out unconditional cmake defines from input +common_cmake_defines = {} +if args.assertions: + common_cmake_defines['LLVM_ENABLE_ASSERTIONS'] = 'ON' +if args.vendor_string: + common_cmake_defines['CLANG_VENDOR'] = args.vendor_string + common_cmake_defines['LLD_VENDOR'] = args.vendor_string +if args.defines: + defines = dict(define.split('=', 1) for define in args.defines) + common_cmake_defines.update(defines) + +# Build bootstrap compiler if user did not request a single stage build +if (use_bootstrap := not args.build_stage1_only): + tc_build.utils.print_header('Building LLVM (bootstrap)') + + bootstrap = LLVMBootstrapBuilder() + bootstrap.ccache = not args.no_ccache + bootstrap.cmake_defines.update(common_cmake_defines) + bootstrap.folders.build = Path(build_folder, 'bootstrap') + bootstrap.folders.source = llvm_folder + bootstrap.quiet_cmake = args.quiet_cmake + bootstrap.show_commands = args.show_build_commands + bootstrap.tools = host_tools + if args.bolt: + bootstrap.projects.append('bolt') + if args.pgo: + bootstrap.projects.append('compiler-rt') + + bootstrap.check_dependencies() + bootstrap.configure() + bootstrap.build() + +# If the user did not specify CMAKE_C_FLAGS or CMAKE_CXX_FLAGS, add them as empty +# to paste stage 2 to ensure there are no environment issues (since CFLAGS and CXXFLAGS +# are taken into account by cmake) +c_flag_defines = ['CMAKE_C_FLAGS', 'CMAKE_CXX_FLAGS'] +for define in c_flag_defines: + if define not in common_cmake_defines: + common_cmake_defines[define] = '' +# The user's build type should be taken into account past the bootstrap compiler +if args.build_type: + common_cmake_defines['CMAKE_BUILD_TYPE'] = args.build_type + +if args.pgo: + if args.full_toolchain: + instrumented = LLVMInstrumentedBuilder() else: - header_string = "stage %d" % stage - sub_folder = "stage%d" % stage - - return (header_string, sub_folder) - - -def invoke_cmake(args, dirs, env_vars, stage): - """ - Invoke cmake to generate the build files - :param args: The args variable generated by parse_parameters - :param dirs: An instance of the Directories class with the paths to use - :param env_vars: An instance of the EnvVars class with the compilers/linker to use - :param stage: What stage we are at - :return: - """ - # Add the defines, point them to our build folder, and invoke cmake - cmake = ['cmake', '-G', 'Ninja', '-Wno-dev'] - defines = build_cmake_defines(args, dirs, env_vars, stage) - for key in defines: - newdef = '-D' + key + '=' + defines[key] - cmake += [newdef] - if args.defines: - for d in args.defines: - cmake += ['-D' + d] - cmake += [dirs.llvm_folder.joinpath("llvm").as_posix()] - - header_string, sub_folder = get_pgo_header_folder(stage) - - cwd = dirs.build_folder.joinpath(sub_folder).as_posix() - - utils.print_header("Configuring LLVM %s" % header_string) - - show_command(args, cmake) - subprocess.run(cmake, check=True, cwd=cwd) - - -def print_install_info(install_folder): - """ - Prints out where the LLVM toolchain is installed, how to add to PATH, and version information - :param install_folder: Where the LLVM toolchain is installed - :return: - """ - bin_folder = install_folder.joinpath("bin") - print("\nLLVM toolchain installed to: %s" % install_folder.as_posix()) - print("\nTo use, either run:\n") - print(" $ export PATH=%s:${PATH}\n" % bin_folder.as_posix()) - print("or add:\n") - print(" PATH=%s:${PATH}\n" % bin_folder.as_posix()) - print("to the command you want to use this toolchain.\n") - - clang = bin_folder.joinpath("clang") - lld = bin_folder.joinpath("ld.lld") - if clang.exists() or lld.exists(): - print("Version information:\n") - for binary in [clang, lld]: - if binary.exists(): - subprocess.run([binary, "--version"], check=True) - print() - - -def ninja_check(args, build_folder): - """ - Invoke ninja with check targets if they are present - :param args: The args variable generated by parse_parameters - :param build_folder: The build folder that ninja should be run in - :return: - """ - if args.check_targets: - ninja_check = ['ninja'] + ['check-%s' % s for s in args.check_targets] - show_command(args, ninja_check) - subprocess.run(ninja_check, check=True, cwd=build_folder) - - -def invoke_ninja(args, dirs, stage): - """ - Invoke ninja to run the actual build - :param args: The args variable generated by parse_parameters - :param dirs: An instance of the Directories class with the paths to use - :param stage: The current stage we're building - :return: - """ - header_string, sub_folder = get_pgo_header_folder(stage) - - utils.print_header("Building LLVM %s" % header_string) - - build_folder = dirs.build_folder.joinpath(sub_folder) - - install_folder = None - if should_install_toolchain(args, stage): - install_folder = dirs.install_folder - elif stage == 1 and args.build_stage1_only and not args.install_stage1_only: - install_folder = build_folder - - build_folder = build_folder.as_posix() - - time_started = time.time() - - show_command(args, ["ninja"]) - subprocess.run('ninja', check=True, cwd=build_folder) - - if stage == get_final_stage(args): - ninja_check(args, build_folder) - - print() - print("LLVM build duration: " + - str(datetime.timedelta(seconds=int(time.time() - time_started)))) - - if should_install_toolchain(args, stage): - subprocess.run(['ninja', 'install'], - check=True, - cwd=build_folder, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - - utils.create_gitignore(install_folder) - - if args.bolt: - do_bolt(args, dirs) - - if install_folder is not None: - print_install_info(install_folder) - - -def get_host_llvm_target(): - """ - Get the host's LLVM target based on 'uname -m' - :return: Host's LLVM target - """ - host_mapping = { - "aarch64": "AArch64", - "armv7l": "ARM", - "i386": "X86", - "mips": "Mips", - "mips64": "Mips", - "ppc": "PowerPC", - "ppc64": "PowerPC", - "ppc64le": "PowerPC", - "riscv32": "RISCV", - "riscv64": "RISCV", - "s390x": "SystemZ", - "x86_64": "X86" - } - return host_mapping.get(platform.machine()) - - -def kernel_build_sh(args, config, dirs, profile_type): - """ - Run kernel/build.sh to generate PGO or BOLT profiles - :param args: The args variable generated by parse_parameters - :param config: The config to build (defconfig, allmodconfig, allyesconfig) - :param dirs: An instance of the Directories class with the paths to use - :param profile_type: The type of profile we are building (bolt-instrumentation, bolt-sampling, pgo, or pgo-slim) - :return: - """ - - # Run kernel/build.sh - build_sh = [ - dirs.root_folder.joinpath("kernel", "build.sh"), - '--{}'.format(profile_type) - ] - - targets = get_targets(args) - if "bolt" in profile_type or "slim" in profile_type: - # For BOLT or "slim" PGO, we limit the number of kernels we build for - # each mode: - # - # When using perf, building too many kernels will generate a gigantic - # perf profile. perf2bolt calls 'perf script', which will load the - # entire profile into memory, which could cause OOM for most machines - # and long processing times for the ones that can handle it for little - # extra gain. - # - # With BOLT instrumentation, we generate one profile file for each - # invocation of clang (PID) to avoid profiling just the driver, so - # building multiple kernels will generate a few hundred gigabytes of - # fdata files. - # - # Just do a native build if the host target is in the list of targets - # or the first target if not. - host_target = get_host_llvm_target() - if targets == "all" or host_target in targets: - targets = host_target - else: - targets = targets.split(";")[0] - - build_sh += ['-t', targets] - - if profile_type == "bolt-sampling": - build_sh = [ - "perf", "record", "--branch-filter", "any,u", "--event", - "cycles:u", "--output", - dirs.build_folder.joinpath("perf.data"), "--" - ] + build_sh - - if "bolt" in profile_type: - build_sh += ['-i', dirs.install_folder] - if "pgo" in profile_type: - build_sh += ['-b', dirs.build_folder] - - if config != "defconfig": - build_sh += ['--%s' % config] - - if dirs.linux_folder: - build_sh += ['-k', dirs.linux_folder.as_posix()] - - show_command(args, build_sh) - subprocess.run(build_sh, check=True, cwd=dirs.build_folder.as_posix()) - - -def pgo_llvm_build(args, dirs): - """ - Builds LLVM as a PGO benchmark - :param args: The args variable generated by parse_parameters - :param dirs: An instance of the Directories class with the paths to use - :return: - """ - # Run check targets if the user requested them for PGO coverage - ninja_check(args, dirs.build_folder.joinpath("stage2").as_posix()) - # Then, build LLVM as if it were the full final toolchain - stage = "pgo" - dirs.build_folder.joinpath(stage).mkdir(parents=True, exist_ok=True) - invoke_cmake(args, dirs, None, stage) - invoke_ninja(args, dirs, stage) - - -def generate_pgo_profiles(args, dirs): - """ - Build a set of kernels across a few architectures to generate PGO profiles - :param args: The args variable generated by parse_parameters - :param dirs: An instance of the Directories class with the paths to use - :return: - """ - - utils.print_header("Building PGO profiles") - - # Run PGO benchmarks - for pgo in args.pgo: - pgo = pgo.split("-") - pgo_target = pgo[0] - if pgo_target == "kernel": - pgo_config = pgo[1] - # Handle "kernel-defconfig-slim" - if len(pgo) == 3: - pgo_type = "pgo-slim" + instrumented = LLVMSlimInstrumentedBuilder() + instrumented.cmake_defines.update(common_cmake_defines) + # We run the tests on the instrumented stage if the LLVM benchmark was enabled + instrumented.check_targets = args.check_targets if 'llvm' in args.pgo else None + instrumented.folders.build = Path(build_folder, 'instrumented') + instrumented.folders.source = llvm_folder + instrumented.projects = final.projects + instrumented.quiet_cmake = args.quiet_cmake + instrumented.show_commands = args.show_build_commands + instrumented.targets = final.targets + instrumented.tools = StageTools(Path(bootstrap.folders.build, 'bin')) + + tc_build.utils.print_header('Building LLVM (instrumented)') + instrumented.configure() + instrumented.build() + + tc_build.utils.print_header('Generating PGO profiles') + pgo_builders = [] + if 'llvm' in args.pgo: + llvm_builder = def_llvm_builder_cls() + llvm_builder.cmake_defines.update(common_cmake_defines) + llvm_builder.folders.build = Path(build_folder, 'profiling') + llvm_builder.folders.source = llvm_folder + llvm_builder.projects = final.projects + llvm_builder.quiet_cmake = args.quiet_cmake + llvm_builder.show_commands = args.show_build_commands + llvm_builder.targets = final.targets + llvm_builder.tools = StageTools(Path(instrumented.folders.build, 'bin')) + pgo_builders.append(llvm_builder) + + # If the user specified both a full and slim build of the same type, remove + # the full build and warn them. + pgo_targets = [s.replace('kernel-', '') for s in args.pgo if 'kernel-' in s] + for pgo_target in pgo_targets: + if 'slim' not in pgo_target: + continue + config_target = pgo_target.split('-')[0] + if config_target in pgo_targets: + tc_build.utils.print_warning( + f"Both full and slim were specified for {config_target}, ignoring full...") + pgo_targets.remove(config_target) + + if pgo_targets: + kernel_builder = LLVMKernelBuilder() + kernel_builder.folders.build = Path(build_folder, 'linux') + kernel_builder.folders.source = lsm.location + kernel_builder.toolchain_prefix = instrumented.folders.build + for item in pgo_targets: + pgo_target = item.split('-') + + config_target = pgo_target[0] + # For BOLT or "slim" PGO, we limit the number of kernels we build for + # each mode: + # + # When using perf, building too many kernels will generate a gigantic + # perf profile. perf2bolt calls 'perf script', which will load the + # entire profile into memory, which could cause OOM for most machines + # and long processing times for the ones that can handle it for little + # extra gain. + # + # With BOLT instrumentation, we generate one profile file for each + # invocation of clang (PID) to avoid profiling just the driver, so + # building multiple kernels will generate a few hundred gigabytes of + # fdata files. + # + # Just do a native build if the host target is in the list of targets + # or the first target if not. + if len(pgo_target) == 2: # slim + if instrumented.host_target_is_enabled(): + llvm_targets = [instrumented.host_target()] + else: + llvm_targets = final.targets[0:1] + # full + elif 'all' in final.targets: + llvm_targets = llvm_source.default_targets() else: - pgo_type = "pgo" - kernel_build_sh(args, pgo_config, dirs, pgo_type) - if pgo_target == "llvm": - pgo_llvm_build(args, dirs) - - # Combine profiles - subprocess.run([ - dirs.build_folder.joinpath("stage1", "bin", "llvm-profdata"), "merge", - "-output=%s" % dirs.build_folder.joinpath("profdata.prof").as_posix() - ] + glob.glob( - dirs.build_folder.joinpath("stage2", "profiles", - "*.profraw").as_posix()), - check=True) - - -def do_multistage_build(args, dirs, env_vars): - stages = [1] - - if not args.build_stage1_only: - stages += [2] - if args.pgo: - stages += [3] - - for stage in stages: - dirs.build_folder.joinpath("stage%d" % stage).mkdir(parents=True, - exist_ok=True) - invoke_cmake(args, dirs, env_vars, stage) - invoke_ninja(args, dirs, stage) - # Build profiles after stage 2 when using PGO - if instrumented_stage(args, stage): - generate_pgo_profiles(args, dirs) - - -def can_use_perf(): - """ - Checks if perf can be used for branch sampling with BOLT - :return: True if perf can be used for branch sampling with BOLT, false if not - """ - # Make sure perf is in the environment - if shutil.which("perf"): - try: - subprocess.run([ - "perf", "record", "--branch-filter", "any,u", "--event", - "cycles:u", "--output", "/dev/null", "--", "sleep", "1" - ], - stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - check=True) - except: - pass - else: - return True - - return False - - -def do_bolt(args, dirs): - # Default to instrumentation, as it should be universally available. - mode = "instrumentation" - - # If we can use perf for branch sampling, we switch to that mode, as - # it is much quicker and it can result in more performance gains - if can_use_perf(): - mode = "sampling" - - utils.print_header("Performing BOLT with {}".format(mode)) - - # clang-#: original binary - # clang.bolt: BOLT optimized binary - # .bolt will become original binary after optimization - clang = dirs.install_folder.joinpath("bin", "clang").resolve() - clang_bolt = clang.with_name("clang.bolt") - - llvm_bolt = dirs.build_folder.joinpath("stage1", "bin", "llvm-bolt") - bolt_profile = dirs.build_folder.joinpath("clang.fdata") - - if mode == "instrumentation": - # clang.inst: instrumented binary, will be removed after generating profiles - clang_inst = clang.with_name("clang.inst") - - # Remove profile if it exists (incremental builds). The missing_ok - # parameter to unlink() was only introduced in Python 3.8 but we try to - # support older versions when possible. - try: - bolt_profile.unlink() - except FileNotFoundError: - pass - - # Instrument clang - clang_inst_cmd = [ - llvm_bolt, "--instrument", - "--instrumentation-file={}".format(bolt_profile), - "--instrumentation-file-append-pid", "-o", clang_inst, clang - ] - show_command(args, clang_inst_cmd) - subprocess.run(clang_inst_cmd, check=True) - - # Generate profile data by using clang to build kernels - kernel_build_sh(args, "defconfig", dirs, "bolt-{}".format(mode)) - - # With instrumentation, we need to combine the profiles we generated, as - # they are separated by PID - if mode == "instrumentation": - merge_fdata = dirs.build_folder.joinpath("stage1", "bin", - "merge-fdata") - fdata_files = glob.glob("{}.*.fdata".format(bolt_profile.as_posix())) - - # merge-fdata will print one line for each .fdata it merges. Redirect - # the output to a log file in case it ever needs to be inspected - merge_fdata_log = dirs.build_folder.joinpath("merge-fdata.log") - - with open(bolt_profile, "w") as out_f, open(merge_fdata_log, - "w") as err_f: - # We don't use show_command() here because of how long the command - # will be. - print("Merging .fdata files, this might take a while...") - subprocess.run([merge_fdata] + fdata_files, - stdout=out_f, - stderr=err_f, - check=True) - - for fdata_file in fdata_files: - os.remove(fdata_file) - - # If we generated perf data, we need to convert it into something BOLT can - # understand. This is already done with instrumentation. - if mode == "sampling": - perf2bolt = dirs.build_folder.joinpath("stage1", "bin", "perf2bolt") - perf_data = dirs.build_folder.joinpath("perf.data") - - perf2bolt_cmd = [perf2bolt, "-p", perf_data, "-o", bolt_profile, clang] - show_command(args, perf2bolt_cmd) - subprocess.run(perf2bolt_cmd, check=True) - - try: - perf_data.unlink() - except FileNotFoundError: - pass - - # Generate BOLT optimized clang - # Flags are from https://github.com/llvm/llvm-project/blob/2696d82fa0c323d92d8794f0a34ea9619888fae9/bolt/docs/OptimizingClang.md - clang_opt_cmd = [ - llvm_bolt, "--data={}".format(bolt_profile), "--reorder-blocks=cache+", - "--reorder-functions=hfsort+", "--split-functions=3", - "--split-all-cold", "--dyno-stats", "--icf=1", "--use-gnu-stack", "-o", - clang_bolt, clang - ] - show_command(args, clang_opt_cmd) - subprocess.run(clang_opt_cmd, check=True) - - # Move BOLT optimized clang into place and remove instrumented clang - shutil.move(clang_bolt, clang) - if mode == "instrumentation": - clang_inst.unlink() - - -def main(): - root_folder = pathlib.Path(__file__).resolve().parent - - args = parse_parameters(root_folder) - - # There are a couple of known issues with BOLT in instrumentation mode: - # https://github.com/llvm/llvm-project/issues/55004 - # https://github.com/llvm/llvm-project/issues/55005 - # Warn the user about them - if args.bolt and not can_use_perf(): - warn = False - - if args.pgo and not args.assertions: - utils.print_warning( - "\nUsing BOLT in instrumentation mode with PGO and no assertions might result in a binary that crashes:" - ) - utils.print_warning( - "https://github.com/llvm/llvm-project/issues/55004") - utils.print_warning( - "Consider adding '--assertions' if there are any failures during the BOLT stage." - ) - warn = True - - if platform.machine() != "x86_64": - utils.print_warning( - "\nUsing BOLT in instrumentation mode may not work on non-x86_64 machines:" - ) - utils.print_warning( - "https://github.com/llvm/llvm-project/issues/55005") - utils.print_warning( - "Consider dropping '--bolt' if there are any failures during the BOLT stage." - ) - warn = True - - if warn: - utils.print_warning( - "Continuing in 5 seconds, hit Ctrl-C to cancel...") - time.sleep(5) - - build_folder = pathlib.Path(args.build_folder) - if not build_folder.is_absolute(): - build_folder = root_folder.joinpath(build_folder) - - install_folder = pathlib.Path(args.install_folder) - if not install_folder.is_absolute(): - install_folder = root_folder.joinpath(install_folder) - - linux_folder = None - if args.linux_folder: - linux_folder = pathlib.Path(args.linux_folder) - if not linux_folder.is_absolute(): - linux_folder = root_folder.joinpath(linux_folder) - if not linux_folder.exists(): - utils.print_error("\nSupplied kernel source (%s) does not exist!" % - linux_folder.as_posix()) - exit(1) - - env_vars = EnvVars(*check_cc_ld_variables(root_folder)) - check_dependencies() - if args.use_good_revision: - ref = GOOD_REVISION - else: - ref = args.branch - - if args.llvm_folder: - llvm_folder = pathlib.Path(args.llvm_folder) - if not llvm_folder.is_absolute(): - llvm_folder = root_folder.joinpath(llvm_folder) - if not llvm_folder.exists(): - utils.print_error("\nSupplied LLVM source (%s) does not exist!" % - linux_folder.as_posix()) - exit(1) + llvm_targets = final.targets + + kernel_builder.matrix[config_target] = llvm_targets + + pgo_builders.append(kernel_builder) + + for pgo_builder in pgo_builders: + if hasattr(pgo_builder, 'configure') and callable(pgo_builder.configure): + tc_build.utils.print_info('Building LLVM for profiling...') + pgo_builder.configure() + pgo_builder.build() + + instrumented.generate_profdata() + +# Final build +final.check_targets = args.check_targets +final.cmake_defines.update(common_cmake_defines) +final.folders.build = Path(build_folder, 'final') +final.folders.install = Path(args.install_folder).resolve() if args.install_folder else None +final.install_targets = args.install_targets +final.quiet_cmake = args.quiet_cmake +final.show_commands = args.show_build_commands + +if args.lto: + final.cmake_defines['LLVM_ENABLE_LTO'] = args.lto.capitalize() +if args.pgo: + final.cmake_defines['LLVM_PROFDATA_FILE'] = Path(instrumented.folders.build, 'profdata.prof') + +if use_bootstrap: + final.tools = StageTools(Path(bootstrap.folders.build, 'bin')) +else: + # If we skipped bootstrapping, we need to check the dependencies now + # and pass along certain user options + final.check_dependencies() + final.ccache = not args.no_ccache + final.tools = host_tools + + # If the user requested BOLT but did not specify it in their projects nor + # bootstrapped, we need to enable it to get the tools we need. + if args.bolt: + if not ('all' in final.projects or 'bolt' in final.projects): + final.projects.append('bolt') + final.tools.llvm_bolt = Path(final.folders.build, 'bin/llvm-bolt') + final.tools.merge_fdata = Path(final.folders.build, 'bin/merge-fdata') + final.tools.perf2bolt = Path(final.folders.build, 'bin/perf2bolt') + +if args.bolt: + final.bolt = True + final.bolt_builder = LLVMKernelBuilder() + final.bolt_builder.folders.build = Path(build_folder, 'linux') + final.bolt_builder.folders.source = lsm.location + if final.host_target_is_enabled(): + llvm_targets = [final.host_target()] else: - llvm_folder = root_folder.joinpath("llvm-project") - fetch_llvm_binutils(root_folder, llvm_folder, not args.no_update, - args.shallow_clone, ref) - cleanup(build_folder, args.incremental) - dirs = Directories(build_folder, install_folder, linux_folder, llvm_folder, - root_folder) - do_multistage_build(args, dirs, env_vars) + llvm_targets = final.targets[0:1] + final.bolt_builder.matrix['defconfig'] = llvm_targets +tc_build.utils.print_header('Building LLVM (final)') +final.configure() +final.build() +final.show_install_info() -if __name__ == '__main__': - main() +print(f"Script duration: {tc_build.utils.get_duration(script_start)}") diff --git a/build-tc.sh b/build-tc.sh index db98ab5..85c59b1 100755 --- a/build-tc.sh +++ b/build-tc.sh @@ -56,7 +56,7 @@ tg_post_msg "$LLVM_NAME: Building LLVM. . ." --clang-vendor "$LLVM_NAME" \ --projects "clang;lld" \ --targets X86 \ - --branch "release/14.x" + --branch "release/16.x" --shallow-clone \ --incremental \ --build-type "Release" 2>&1 | tee build.log diff --git a/ci.sh b/ci.sh index 9ab8dad..2638de8 100755 --- a/ci.sh +++ b/ci.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash base=$(dirname "$(readlink -f "$0")") +install=$base/install +src=$base/src set -eu @@ -22,7 +24,10 @@ function do_all() { } function do_binutils() { - "$base"/build-binutils.py -t x86_64 + "$base"/build-binutils.py \ + --install-folder "$install" \ + --show-build-commands \ + --targets x86_64 } function do_deps() { @@ -53,8 +58,34 @@ function do_deps() { } function do_kernel() { - cd "$base"/kernel - ./build.sh -t X86 + local branch=linux-rolling-stable + local linux=$src/$branch + + if [[ -d $linux ]]; then + git -C "$linux" fetch --depth=1 origin $branch + git -C "$linux" reset --hard FETCH_HEAD + else + git clone \ + --branch "$branch" \ + --depth=1 \ + --single-branch \ + https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git \ + "$linux" + fi + + cat </dev/null - __clang_major__ __clang_minor__ __clang_patchlevel__ - EOF -} - -# Convert the version string x.y.z to a canonical 5 or 6-digit form. -get_canonical_version() { - IFS=. - set -- $1 - echo $((10000 * $1 + 100 * $2 + $3)) -} - -# $@ instead of $1 because multiple words might be given, e.g. CC="ccache gcc". -set -- $(get_compiler_info "$@") - -get_canonical_version $1.$2.$3 diff --git a/kernel/.gitignore b/kernel/.gitignore deleted file mode 100644 index b19962c..0000000 --- a/kernel/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -linux-* -!*.patch -!*.sha256 diff --git a/kernel/build.sh b/kernel/build.sh deleted file mode 100755 index af0b5a3..0000000 --- a/kernel/build.sh +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env bash - -krnl=$(dirname "$(readlink -f "$0")") -tc_bld=${krnl%/*} - -function header() { - border="===$(for _ in $(seq ${#1}); do printf '='; done)===" - printf '\033[1m\n%s\n%s\n%s\n\n\033[0m' "$border" "== $1 ==" "$border" -} - -# Parse parameters -function parse_parameters() { - targets=() - while (($#)); do - case $1 in - --allmodconfig) - config_target=allmodconfig - ;; - - --allyesconfig) - config_target=allyesconfig - ;; - - -b | --build-folder) - shift - build_folder=$1 - ;; - - --bolt-*) - bolt=$(echo "$1" | cut -d - -f 4) - ;; - - -i | --install-folder) - shift - install_folder=$1 - ;; - - -k | --kernel-src) - shift - kernel_src=$1 - ;; - - -p | --path-override) - shift - path_override=$1 - ;; - - --pgo) - pgo=true - ;; - - -t | --targets) - shift - IFS=";" read -ra llvm_targets <<<"$1" - # Convert LLVM targets into GNU triples - for llvm_target in "${llvm_targets[@]}"; do - case $llvm_target in - AArch64) targets+=(aarch64-linux-gnu) ;; - ARM) targets+=(arm-linux-gnueabi) ;; - Hexagon) targets+=(hexagon-linux-gnu) ;; - Mips) targets+=(mipsel-linux-gnu) ;; - PowerPC) targets+=(powerpc-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu) ;; - RISCV) targets+=(riscv64-linux-gnu) ;; - SystemZ) targets+=(s390x-linux-gnu) ;; - X86) targets+=(x86_64-linux-gnu) ;; - esac - done - ;; - esac - shift - done -} - -function set_default_values() { - [[ -z ${targets[*]} || ${targets[*]} = "all" ]] && targets=( - arm-linux-gnueabi - aarch64-linux-gnu - hexagon-linux-gnu - mipsel-linux-gnu - powerpc-linux-gnu - powerpc64-linux-gnu - powerpc64le-linux-gnu - riscv64-linux-gnu - s390x-linux-gnu - x86_64-linux-gnu - ) - [[ -z $config_target ]] && config_target=defconfig -} - -function setup_up_path() { - # Add the default install bin folder to PATH for binutils - export PATH=$tc_bld/install/bin:$PATH - - # Add the stage 2 bin folder to PATH for the instrumented clang if we are doing PGO - ${pgo:=false} && export PATH=${build_folder:=$tc_bld/build/llvm}/stage2/bin:$PATH - - # Add the user's install folder if it is not in the PATH already if we are doing bolt - if [[ -n $bolt ]] && [[ -n $install_folder ]]; then - install_bin=$install_folder/bin - echo "$PATH" | grep -q "$install_bin:" || export PATH=$install_bin:$PATH - fi - - # If the user wants to add another folder to PATH, they can do it with the path_override variable - [[ -n $path_override ]] && export PATH=$path_override:$PATH -} - -# Turns 'patch -N' from a fatal error to an informational message -function apply_patch { - patch_file=${1:?} - if ! patch_out=$(patch -Np1 <"$patch_file"); then - patch_out_ok=$(echo "$patch_out" | grep "Reversed (or previously applied) patch detected") - if [[ -n $patch_out_ok ]]; then - echo "${patch_file##*/}: $patch_out_ok" - else - echo "$patch_out" - exit 2 - fi - fi -} - -function setup_krnl_src() { - # A kernel folder can be supplied via '-k' for testing the script - if [[ -n $kernel_src ]]; then - cd "$kernel_src" || exit - else - linux="linux-5.18" - linux_tarball=$krnl/$linux.tar.xz - - # If we don't have the source tarball, download and verify it - if [[ ! -f $linux_tarball ]]; then - curl -LSso "$linux_tarball" https://cdn.kernel.org/pub/linux/kernel/v5.x/"${linux_tarball##*/}" - - ( - cd "${linux_tarball%/*}" || exit - sha256sum -c "$linux_tarball".sha256 --quiet - ) || { - echo "Linux tarball verification failed! Please remove '$linux_tarball' and try again." - exit 1 - } - fi - - # If there is a patch to apply, remove the folder so that we can patch it accurately (we cannot assume it has already been patched) - patch_files=() - for src_file in "$krnl"/*; do - [[ ${src_file##*/} = *.patch ]] && patch_files+=("$src_file") - done - [[ -n "${patch_files[*]}" ]] && rm -rf $linux - [[ -d $linux ]] || { tar -xf "$linux_tarball" || exit; } - cd $linux || exit - for patch_file in "${patch_files[@]}"; do - apply_patch "$patch_file" - done - fi -} - -# Can the requested architecture use LLVM_IAS=1? This assumes that if the user -# is passing in their own kernel source via '-k', it is either the same or a -# newer version as the one that the script downloads to avoid having a two -# variable matrix. -function can_use_llvm_ias() { - local llvm_version - llvm_version=$("$tc_bld"/clang-version.sh clang) - - case $1 in - # https://github.com/ClangBuiltLinux/linux/issues?q=is%3Aissue+label%3A%22%5BARCH%5D+arm32%22+label%3A%22%5BTOOL%5D+integrated-as%22+ - arm*) - if [[ $llvm_version -ge 130000 ]]; then - return 0 - else - return 1 - fi - ;; - - # https://github.com/ClangBuiltLinux/linux/issues?q=is%3Aissue+label%3A%22%5BARCH%5D+arm64%22+label%3A%22%5BTOOL%5D+integrated-as%22+ - # https://github.com/ClangBuiltLinux/linux/issues?q=is%3Aissue+label%3A%22%5BARCH%5D+x86_64%22+label%3A%22%5BTOOL%5D+integrated-as%22+ - aarch64* | x86_64*) - if [[ $llvm_version -ge 110000 ]]; then - return 0 - else - return 1 - fi - ;; - - hexagon* | mips* | riscv*) - # All supported versions of LLVM for building the kernel - return 0 - ;; - - powerpc* | s390*) - # No supported versions of LLVM for building the kernel - return 1 - ;; - esac -} - -# Get as command based on prefix and host architecture. See host_arch_target() -# in build-binutils.py. -function get_as() { - local host_target target_arch - - case "$(uname -m)" in - armv7l) host_target=arm ;; - ppc64) host_target=powerpc64 ;; - ppc64le) host_target=powerpc64le ;; - ppc) host_target=powerpc ;; - *) host_target=$(uname -m) ;; - esac - - # Turn triple (--) into - target_arch=${1%%-*} - - if [[ "$target_arch" = "$host_target" ]]; then - echo "as" - else - echo "$1-as" - fi -} - -function check_binutils() { - # Check for all binutils and build them if necessary - binutils_targets=() - - for prefix in "${targets[@]}"; do - # We do not need to check for binutils if we can use the integrated assembler - can_use_llvm_ias "$prefix" && continue - - command -v "$(get_as "$prefix")" &>/dev/null || binutils_targets+=("$prefix") - done - - [[ -n "${binutils_targets[*]}" ]] && { "$tc_bld"/build-binutils.py -t "${binutils_targets[@]}" || exit; } -} - -function print_tc_info() { - # Print final toolchain information - header "Toolchain information" - clang --version - for prefix in "${targets[@]}"; do - can_use_llvm_ias "$prefix" && continue - - echo - "$(get_as "$prefix")" --version - done -} - -# Checks if clang can be used as a host toolchain. This command will error with -# "No available targets are compatible with triple ..." if clang has been built -# without support for the host target. This is better than keeping a map of -# 'uname -m' against the target's name. -function clang_supports_host_target() { - echo | clang -x c -c -o /dev/null - &>/dev/null -} - -function build_kernels() { - make_base=(make -skj"$(nproc)" KCFLAGS=-Wno-error LLVM=1 O=out) - [[ $bolt = "instrumentation" ]] && make_base+=(CC=clang.inst) - - if clang_supports_host_target; then - [[ $bolt = "instrumentation" ]] && make_base+=(HOSTCC=clang.inst) - else - make_base+=(HOSTCC=gcc HOSTCXX=g++) - fi - - header "Building kernels ($(make -s kernelversion))" - - # If the user has any CFLAGS in their environment, they can cause issues when building tools/ - # Ideally, the kernel would always clobber user flags via ':=' but that is not always the case - unset CFLAGS - - set -x - - for target in "${targets[@]}"; do - make=("${make_base[@]}") - can_use_llvm_ias "$target" || make+=(CROSS_COMPILE="$target-" LLVM_IAS=0) - - case $target in - arm-linux-gnueabi) - case $config_target in - defconfig) - configs=(multi_v5_defconfig aspeed_g5_defconfig multi_v7_defconfig) - ;; - *) - configs=("$config_target") - ;; - esac - for config in "${configs[@]}"; do - time "${make[@]}" \ - ARCH=arm \ - distclean "$config" all || exit - done - ;; - - aarch64-linux-gnu) - time "${make[@]}" \ - ARCH=arm64 \ - distclean "$config_target" all || exit - ;; - - hexagon-linux-gnu) - time "${make[@]}" \ - ARCH=hexagon \ - distclean "$config_target" all || exit - ;; - - mipsel-linux-gnu) - time "${make[@]}" \ - ARCH=mips \ - distclean malta_defconfig all || exit - ;; - - powerpc-linux-gnu) - time "${make[@]}" \ - ARCH=powerpc \ - distclean ppc44x_defconfig all || exit - ;; - - powerpc64-linux-gnu) - time "${make[@]}" \ - ARCH=powerpc \ - LD="$target-ld" \ - distclean pseries_defconfig disable-werror.config all || exit - ;; - - powerpc64le-linux-gnu) - time "${make[@]}" \ - ARCH=powerpc \ - distclean powernv_defconfig all || exit - ;; - - riscv64-linux-gnu) - time "${make[@]}" \ - ARCH=riscv \ - distclean "$config_target" all || exit - ;; - - s390x-linux-gnu) - time "${make[@]}" \ - ARCH=s390 \ - LD="$target-ld" \ - OBJCOPY="$target-objcopy" \ - OBJDUMP="$target-objdump" \ - distclean "$config_target" all || exit - ;; - - x86_64-linux-gnu) - time "${make[@]}" \ - ARCH=x86_64 \ - distclean "$config_target" all || exit - ;; - esac - done -} - -parse_parameters "$@" -set_default_values -setup_up_path -setup_krnl_src -check_binutils -print_tc_info -build_kernels diff --git a/kernel/linux-5.18.tar.xz.sha256 b/kernel/linux-5.18.tar.xz.sha256 deleted file mode 100644 index d219770..0000000 --- a/kernel/linux-5.18.tar.xz.sha256 +++ /dev/null @@ -1 +0,0 @@ -51f3f1684a896e797182a0907299cc1f0ff5e5b51dd9a55478ae63a409855cee linux-5.18.tar.xz diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..9c27f49 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,30 @@ +# https://beta.ruff.rs/docs/rules/ +select = [ + 'A', # flake8-builtins + 'ARG', # flake8-unused-arguments + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'E', # pycodestyle + 'F', # pyflakes + 'PIE', # flake8-pie + 'PL', # pylint + 'PTH', # flake8-use-pathlib + 'RET', # flake8-return + 'RUF', # ruff + 'S', # flake8-bandit + 'SIM', # flake8-simplify + 'SLF', # flake8-self + 'UP', # pyupgrade + 'W', # pycodestyle +] +ignore = [ + 'E501', # line-too-long + 'PLR0911', # too-many-return-statments + 'PLR0912', # too-many-branches + 'PLR0913', # too-many-arguments + 'PLR0915', # too-many-statements + 'PLR2004', # magic-value-comparison + 'S603', # subprocess-without-shell-equals-true + 'S607', # start-process-with-partial-path +] +target-version = 'py38' diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..ac26fc2 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,2 @@ +* +!*.patch diff --git a/tc_build/__init__.py b/tc_build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tc_build/binutils.py b/tc_build/binutils.py new file mode 100644 index 0000000..09eb98c --- /dev/null +++ b/tc_build/binutils.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 + +import os +from pathlib import Path +import platform + +from tc_build.builder import Builder +from tc_build.source import SourceManager +import tc_build.utils + + +class BinutilsBuilder(Builder): + + def __init__(self): + super().__init__() + + self.cflags = ['-O2'] + self.configure_flags = [ + '--disable-compressed-debug-sections', + '--disable-gdb', + '--disable-nls', + '--disable-werror', + '--enable-deterministic-archives', + '--enable-new-dtags', + '--enable-plugins', + '--enable-threads', + '--quiet', + '--with-system-zlib', + ] + # gprofng uses glibc APIs that might not be available on musl + if tc_build.utils.libc_is_musl(): + self.configure_flags.append('--disable-gprofng') + self.configure_vars = { + 'CC': 'gcc', + 'CXX': 'g++', + } + self.extra_targets = [] + self.native_arch = '' + self.target = '' + + def build(self): + if self.folders.install: + self.configure_flags.append(f"--prefix={self.folders.install}") + if platform.machine() != self.native_arch: + self.configure_flags += [ + f"--program-prefix={self.target}-", + f"--target={self.target}", + ] + if self.extra_targets: + self.configure_flags.append(f"--enable-targets={','.join(self.extra_targets)}") + + self.configure_vars['CFLAGS'] = ' '.join(self.cflags) + self.configure_vars['CXXFLAGS'] = ' '.join(self.cflags) + + self.clean_build_folder() + self.folders.build.mkdir(exist_ok=True, parents=True) + tc_build.utils.print_header(f"Building {self.target} binutils") + + configure_cmd = [ + Path(self.folders.source, 'configure'), + *self.configure_flags, + ] + [f"{var}={val}" for var, val in self.configure_vars.items()] + self.run_cmd(configure_cmd, cwd=self.folders.build) + + make_cmd = ['make', '-C', self.folders.build, '-s', f"-j{os.cpu_count()}", 'V=0'] + self.run_cmd(make_cmd) + + if self.folders.install: + self.run_cmd([*make_cmd, 'install']) + tc_build.utils.create_gitignore(self.folders.install) + + +class StandardBinutilsBuilder(BinutilsBuilder): + + def __init__(self): + super().__init__() + + self.configure_flags += [ + '--disable-sim', + '--enable-lto', + '--enable-relro', + '--with-pic', + ] + + +class NoMultilibBinutilsBuilder(BinutilsBuilder): + + def __init__(self): + super().__init__() + + self.configure_flags += [ + '--disable-multilib', + '--with-gnu-as', + '--with-gnu-ld', + ] + + +class ArmBinutilsBuilder(NoMultilibBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'armv7l' + self.target = 'arm-linux-gnueabi' + + +class AArch64BinutilsBuilder(NoMultilibBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'aarch64' + self.target = 'aarch64-linux-gnu' + + +class LoongArchBinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'loongarch64' + self.target = 'loongarch64-linux-gnu' + + +class MipsBinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self, endian_suffix=''): + super().__init__() + + target_64 = f"mips64{endian_suffix}" + self.extra_targets = [f"{target_64}-linux-gnueabi64", f"{target_64}-linux-gnueabin32"] + + target_32 = f"mips{endian_suffix}" + self.native_target = target_32 + self.target = f"{target_32}-linux-gnu" + + +class MipselBinutilsBuilder(MipsBinutilsBuilder): + + def __init__(self): + super().__init__('el') + + +class PowerPCBinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'ppc' + self.target = 'powerpc-linux-gnu' + + +class PowerPC64BinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'ppc64' + self.target = 'powerpc64-linux-gnu' + + +class PowerPC64LEBinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'ppc64le' + self.target = 'powerpc64le-linux-gnu' + + +class RISCV64BinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.native_arch = 'riscv64' + self.target = 'riscv64-linux-gnu' + + +class S390XBinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.extra_targets.append('s390-linux-gnu') + self.native_arch = 's390x' + self.target = 's390x-linux-gnu' + + +class X8664BinutilsBuilder(StandardBinutilsBuilder): + + def __init__(self): + super().__init__() + + self.extra_targets.append('x86_64-pep') + self.native_arch = 'x86_64' + self.target = 'x86_64-linux-gnu' + + +class BinutilsSourceManager(SourceManager): + + def default_targets(self): + targets = [ + 'aarch64', + 'arm', + 'mips', + 'mipsel', + 'powerpc', + 'powerpc64', + 'powerpc64le', + 'riscv64', + 's390x', + 'x86_64', + ] + if Path(self.location, 'gas/config/tc-loongarch.c').exists(): + targets.append('loongarch64') + return targets + + def prepare(self): + if not self.location: + raise RuntimeError('No source location set?') + if self.location.exists(): + return # source already set up + + if not self.tarball.local_location: + raise RuntimeError('No local tarball location set?') + if not self.tarball.local_location.exists(): + self.tarball.download() + + self.tarball.extract(self.location) + tc_build.utils.print_info(f"Source sucessfully prepared in {self.location}") diff --git a/tc_build/builder.py b/tc_build/builder.py new file mode 100644 index 0000000..2aac938 --- /dev/null +++ b/tc_build/builder.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import shlex +import shutil +import subprocess + + +class Folders: + + def __init__(self): + self.build = None + self.install = None + self.source = None + + +class Builder: + + def __init__(self): + self.folders = Folders() + self.show_commands = False + + def build(self): + raise NotImplementedError + + def clean_build_folder(self): + if not self.folders.build: + raise RuntimeError('No build folder set?') + + if self.folders.build.exists(): + if self.folders.build.is_dir(): + shutil.rmtree(self.folders.build) + else: + self.folders.build.unlink() + + def run_cmd(self, cmd, capture_output=False, cwd=None): + if self.show_commands: + # Acts sort of like 'set -x' in bash + print(f"$ {' '.join([shlex.quote(str(elem)) for elem in cmd])}", flush=True) + return subprocess.run(cmd, capture_output=capture_output, check=True, cwd=cwd) diff --git a/tc_build/kernel.py b/tc_build/kernel.py new file mode 100644 index 0000000..6ec22c4 --- /dev/null +++ b/tc_build/kernel.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 + +import os +from pathlib import Path +import shutil +import subprocess +import time + +from tc_build.builder import Builder +from tc_build.source import SourceManager +import tc_build.utils + + +class KernelBuilder(Builder): + + # If the user supplies their own kernel source, it must be at least this + # version to ensure that all the build commands work, as the build commands + # were written to target at least this version. + MINIMUM_SUPPORTED_VERSION = (6, 1, 7) + + def __init__(self, arch): + super().__init__() + + self.bolt_instrumentation = False + self.bolt_sampling_output = None + self.config_targets = None + self.cross_compile = None + self.make_variables = { + 'ARCH': arch, + # We do not want warnings to cause build failures when profiling. + 'KCFLAGS': '-Wno-error', + } + self.show_commands = True + self.toolchain_prefix = None + self.toolchain_version = None + + def build(self): + if not self.toolchain_version: + self.toolchain_version = self.get_toolchain_version() + + if self.bolt_instrumentation: + self.make_variables['CC'] = Path(self.toolchain_prefix, 'bin/clang.inst') + # The user may have configured clang without the host target, in which + # case we need to use GCC for compiling the host utilities. + if self.can_use_clang_as_hostcc(): + if 'CC' in self.make_variables: + self.make_variables['HOSTCC'] = self.make_variables['CC'] + else: + self.make_variables['HOSTCC'] = 'gcc' + self.make_variables['HOSTCXX'] = 'g++' + if self.needs_binutils(): + if not shutil.which(f"{self.cross_compile}elfedit"): + tc_build.utils.print_warning( + f"binutils for {self.make_variables['ARCH']} ('{self.cross_compile}') could not be found, skipping kernel build..." + ) + return + self.make_variables['CROSS_COMPILE'] = self.cross_compile + self.make_variables['LLVM'] = f"{self.toolchain_prefix}/bin/" + if not self.can_use_ias(): + self.make_variables['LLVM_IAS'] = '0' + self.make_variables['O'] = self.folders.build + + make_cmd = [] + if self.bolt_sampling_output: + make_cmd += [ + 'perf', 'record', + '--branch-filter', 'any,u', + '--event', 'cycles:u', + '--output', self.bolt_sampling_output, + '--', + ] # yapf: disable + make_cmd += ['make', '-C', self.folders.source, f"-skj{os.cpu_count()}"] + make_cmd += [f"{key}={self.make_variables[key]}" for key in sorted(self.make_variables)] + make_cmd += [*self.config_targets, 'all'] + + # If the user has any CFLAGS in their environment, they can cause issues when building tools. + # Ideally, the kernel would always clobber user flags via ':=' but we deal with reality. + os.environ.pop('CFLAGS', '') + + self.clean_build_folder() + build_start = time.time() + self.run_cmd(make_cmd) + tc_build.utils.print_info(f"Build duration: {tc_build.utils.get_duration(build_start)}") + + def can_use_ias(self): + return True + + def get_toolchain_version(self): + if not self.toolchain_prefix: + raise RuntimeError('get_toolchain_version(): No toolchain prefix set?') + if not (clang := Path(self.toolchain_prefix, 'bin/clang')).exists(): + raise RuntimeError(f"clang could not be found in {self.toolchain_prefix}?") + + clang_cmd = [clang, '-E', '-P', '-x', 'c', '-'] + clang_input = '__clang_major__ __clang_minor__ __clang_patchlevel__' + clang_output = subprocess.run(clang_cmd, + capture_output=True, + check=True, + input=clang_input, + text=True).stdout.strip() + + return tuple(int(elem) for elem in clang_output.split(' ')) + + def can_use_clang_as_hostcc(self): + clang = Path(self.toolchain_prefix, 'bin/clang') + try: + subprocess.run([clang, '-x', 'c', '-c', '-o', '/dev/null', '/dev/null'], + capture_output=True, + check=True) + except subprocess.CalledProcessError: + return False + return True + + def needs_binutils(self): + return not self.can_use_ias() + + +class ArmKernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('arm') + + self.cross_compile = 'arm-linux-gnueabi-' + + def can_use_ias(self): + return self.toolchain_version >= (13, 0, 0) + + +class ArmV5KernelBuilder(ArmKernelBuilder): + + def __init__(self): + super().__init__() + + self.config_targets = ['multi_v5_defconfig'] + + +class ArmV6KernelBuilder(ArmKernelBuilder): + + def __init__(self): + super().__init__() + + self.config_targets = ['aspeed_g5_defconfig'] + + +class ArmV7KernelBuilder(ArmKernelBuilder): + + def __init__(self): + super().__init__() + + self.config_targets = ['multi_v7_defconfig'] + + +class Arm64KernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('arm64') + + +class HexagonKernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('hexagon') + + +class MIPSKernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('mips') + + self.config_targets = ['malta_defconfig'] + + +class PowerPCKernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('powerpc') + + def can_use_ias(self): + return False + + +class PowerPC32KernelBuilder(PowerPCKernelBuilder): + + def __init__(self): + super().__init__() + + self.config_targets = ['pmac32_defconfig', 'disable-werror.config'] + self.cross_compile = 'powerpc-linux-gnu-' + + +class PowerPC64KernelBuilder(PowerPCKernelBuilder): + + def __init__(self): + super().__init__() + + self.config_targets = ['ppc64_guest_defconfig', 'disable-werror.config'] + self.cross_compile = 'powerpc64-linux-gnu-' + + # https://github.com/ClangBuiltLinux/linux/issues/602 + self.make_variables['LD'] = self.cross_compile + 'ld' + + +class PowerPC64LEKernelBuilder(PowerPCKernelBuilder): + + def __init__(self): + super().__init__() + + self.config_targets = ['powernv_defconfig', 'disable-werror.config'] + self.cross_compile = 'powerpc64le-linux-gnu-' + + def build(self): + self.toolchain_version = self.get_toolchain_version() + # https://github.com/ClangBuiltLinux/linux/issues/1260 + if self.toolchain_version < (12, 0, 0): + self.make_variables['LD'] = self.cross_compile + 'ld' + + super().build() + + # https://github.com/llvm/llvm-project/commit/33504b3bbe10d5d4caae13efcb99bd159c126070 + def can_use_ias(self): + return self.toolchain_version >= (14, 0, 2) + + # https://github.com/ClangBuiltLinux/linux/issues/1601 + def needs_binutils(self): + return True + + +class RISCVKernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('riscv') + + self.cross_compile = 'riscv64-linux-gnu-' + + # https://github.com/llvm/llvm-project/commit/bbea64250f65480d787e1c5ff45c4de3ec2dcda8 + def can_use_ias(self): + return self.toolchain_version >= (13, 0, 0) + + +class S390KernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('s390') + + self.cross_compile = 's390x-linux-gnu-' + + # LD: https://github.com/ClangBuiltLinux/linux/issues/1524 + # OBJCOPY: https://github.com/ClangBuiltLinux/linux/issues/1530 + # OBJDUMP: https://github.com/ClangBuiltLinux/linux/issues/859 + for key in ['LD', 'OBJCOPY', 'OBJDUMP']: + self.make_variables[key] = self.cross_compile + key.lower() + + def build(self): + self.toolchain_version = self.get_toolchain_version() + if self.toolchain_version <= (15, 0, 0): + # https://git.kernel.org/linus/30d17fac6aaedb40d111bb159f4b35525637ea78 + tc_build.utils.print_warning( + 's390 does not build with LLVM < 15.0.0, skipping build...') + return + + super().build() + + def can_use_ias(self): + return True + + def needs_binutils(self): + return True + + +class X8664KernelBuilder(KernelBuilder): + + def __init__(self): + super().__init__('x86_64') + + +class LLVMKernelBuilder(Builder): + + def __init__(self): + super().__init__() + + self.bolt_instrumentation = False + self.bolt_sampling_output = None + self.matrix = {} + self.toolchain_prefix = None + + def build(self): + builders = [] + + allconfig_capable_builders = { + 'AArch64': Arm64KernelBuilder, + 'ARM': ArmKernelBuilder, + 'Hexagon': HexagonKernelBuilder, + 'RISCV': RISCVKernelBuilder, + 'SystemZ': S390KernelBuilder, + 'X86': X8664KernelBuilder, + } + + # This is a little convoluted :/ + # The overall idea here is to avoid duplicating builds, so the + # matrix consists of a series of configuration targets ("defconfig", + # "allmodconfig", etc) and a list of LLVM targets to build for each + # configuration target. From there, this block filters out the + # architectures that cannot build their "all*configs" with clang, so + # they are duplicated if both "defconfig" and "allmodconfig" are + # requested. + for config_target, llvm_targets in self.matrix.items(): + for llvm_target in llvm_targets: + if config_target == 'defconfig' and llvm_target == 'ARM': + builders += [ + ArmV5KernelBuilder(), + ArmV6KernelBuilder(), + ArmV7KernelBuilder(), + ] + elif config_target == 'defconfig' and llvm_target == 'Mips': + builders.append(MIPSKernelBuilder()) + elif config_target == 'defconfig' and llvm_target == 'PowerPC': + builders += [ + PowerPC32KernelBuilder(), + PowerPC64KernelBuilder(), + PowerPC64LEKernelBuilder(), + ] + elif llvm_target in allconfig_capable_builders: + builder = allconfig_capable_builders[llvm_target]() + builder.config_targets = [config_target] + builders.append(builder) + + lsm = LinuxSourceManager() + lsm.location = self.folders.source + tc_build.utils.print_info(f"Building Linux {lsm.get_kernelversion()} for profiling...") + + for builder in builders: + builder.bolt_instrumentation = self.bolt_instrumentation + builder.bolt_sampling_output = self.bolt_sampling_output + builder.folders.build = self.folders.build + builder.folders.source = self.folders.source + builder.toolchain_prefix = self.toolchain_prefix + builder.build() + + +class LinuxSourceManager(SourceManager): + + def __init__(self, location=None): + super().__init__(location) + + self.patches = [] + + def get_kernelversion(self): + return subprocess.run(['make', '-s', 'kernelversion'], + capture_output=True, + check=True, + cwd=self.location, + text=True).stdout.strip() + + # Dynamically get the version of the supplied kernel source as a tuple, + # which can be used to check if a provided kernel source is at least a + # particular version. + def get_version(self): + # elem.split('-')[0] in case we are dealing with an -rc release. + return tuple(int(elem.split('-')[0]) for elem in self.get_kernelversion().split('.', 3)) + + def prepare(self): + self.tarball.download() + # If patches are specified, remove the source folder, we cannot assume + # it has already been patched. + if self.patches: + shutil.rmtree(self.location, ignore_errors=True) + if not self.location.exists(): + self.tarball.extract(self.location) + for patch in self.patches: + patch_cmd = [ + 'patch', + f"--directory={self.location}", + '--forward', + f"--input={patch}", + '--strip=1', + ] + try: + subprocess.run(patch_cmd, capture_output=True, check=True, text=True) + except subprocess.CalledProcessError as err: + # Turns 'patch -N' into a warning versus a hard error; it is + # not the user's fault if we forget to drop a patch that has + # been applied. + if 'Reversed (or previously applied) patch detected' in err.stdout: + tc_build.utils.print_warning( + f"Patch ('{patch}') has already been applied, consider removing it") + else: + raise err + tc_build.utils.print_info(f"Source sucessfully prepared in {self.location}") diff --git a/tc_build/llvm.py b/tc_build/llvm.py new file mode 100644 index 0000000..4ea6447 --- /dev/null +++ b/tc_build/llvm.py @@ -0,0 +1,547 @@ +#!/usr/bin/env python3 + +import contextlib +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import time + +from tc_build.builder import Builder +import tc_build.utils + + +def get_all_targets(llvm_folder): + contents = Path(llvm_folder, 'llvm/CMakeLists.txt').read_text(encoding='utf-8') + if not (match := re.search(r'set\(LLVM_ALL_TARGETS([\w|\s]+)\)', contents)): + raise RuntimeError('Could not find LLVM_ALL_TARGETS?') + return [val for target in match.group(1).splitlines() if (val := target.strip())] + + +class LLVMBuilder(Builder): + + def __init__(self): + super().__init__() + + self.bolt = False + self.bolt_builder = None + self.ccache = False + self.check_targets = [] + # Removes system dependency on terminfo to keep the dynamic library + # dependencies slim. This can be unconditionally done as it does not + # impact clang's ability to show colors for certain output like + # warnings. + self.cmake_defines = {'LLVM_ENABLE_TERMINFO': 'OFF'} + self.install_targets = [] + self.tools = None + self.projects = [] + self.quiet_cmake = False + self.targets = [] + + def bolt_clang(self): + # Default to instrumentation, as it should be universally available. + mode = 'instrumentation' + # If we can use perf for branch sampling, we switch to that mode, as + # it is much quicker and it can result in more performance gains + if self.can_use_perf(): + mode = 'sampling' + + tc_build.utils.print_header(f"Performing BOLT with {mode}") + + # clang-#: original binary + # clang.bolt: BOLT optimized binary + # .bolt will become original binary after optimization + clang = Path(self.folders.build, 'bin/clang').resolve() + clang_bolt = clang.with_name('clang.bolt') + + bolt_profile = Path(self.folders.build, 'clang.fdata') + + if mode == 'instrumentation': + # clang.inst: instrumented binary, will be removed after generating profiles + clang_inst = clang.with_name('clang.inst') + + clang_inst_cmd = [ + self.tools.llvm_bolt, + '--instrument', + f"--instrumentation-file={bolt_profile}", + '--instrumentation-file-append-pid', + '-o', + clang_inst, + clang, + ] + self.run_cmd(clang_inst_cmd) + + self.bolt_builder.bolt_instrumentation = True + + if mode == 'sampling': + self.bolt_builder.bolt_sampling_output = Path(self.folders.build, 'perf.data') + + self.bolt_builder.toolchain_prefix = self.folders.build + self.bolt_builder.build() + + # With instrumentation, we need to combine the profiles we generated, + # as they are separated by PID + if mode == 'instrumentation': + fdata_files = bolt_profile.parent.glob(f"{bolt_profile.name}.*.fdata") + + # merge-fdata will print one line for each .fdata it merges. + # Redirect the output to a log file in case it ever needs to be + # inspected. + merge_fdata_log = Path(self.folders.build, 'merge-fdata.log') + + with bolt_profile.open('w', encoding='utf-8') as out_file, \ + merge_fdata_log.open('w', encoding='utf-8') as err_file: + tc_build.utils.print_info('Merging .fdata files, this might take a while...') + subprocess.run([self.tools.merge_fdata, *list(fdata_files)], + check=True, + stderr=err_file, + stdout=out_file) + for fdata_file in fdata_files: + fdata_file.unlink() + + if mode == 'sampling': + perf2bolt_cmd = [ + self.tools.perf2bolt, + '-p', + self.bolt_builder.bolt_sampling_output, + '-o', + bolt_profile, + clang, + ] + self.run_cmd(perf2bolt_cmd) + self.bolt_builder.bolt_sampling_output.unlink() + + # Now actually optimize clang + clang_opt_cmd = [ + self.tools.llvm_bolt, + f"--data={bolt_profile}", + '--dyno-stats', + '--icf=1', + '-o', + clang_bolt, + '--reorder-blocks=cache+', + '--reorder-functions=hfsort+', + '--split-all-cold', + '--split-functions=3', + '--use-gnu-stack', + clang, + ] + self.run_cmd(clang_opt_cmd) + clang_bolt.replace(clang) + if mode == 'instrumentation': + clang_inst.unlink() + + def build(self): + if not self.folders.build: + raise RuntimeError('No build folder set for build()?') + if not Path(self.folders.build, 'build.ninja').exists(): + raise RuntimeError('No build.ninja in build folder, run configure()?') + if self.bolt and not self.bolt_builder: + raise RuntimeError('BOLT requested without a builder?') + + build_start = time.time() + ninja_cmd = ['ninja', '-C', self.folders.build] + self.run_cmd(ninja_cmd) + + if self.check_targets: + check_targets = [f"check-{target}" for target in self.check_targets] + self.run_cmd([*ninja_cmd, *check_targets]) + + tc_build.utils.print_info(f"Build duration: {tc_build.utils.get_duration(build_start)}") + + if self.bolt: + self.bolt_clang() + + if self.folders.install: + if self.install_targets: + install_targets = [f"install-{target}" for target in self.install_targets] + else: + install_targets = ['install'] + self.run_cmd([*ninja_cmd, *install_targets], capture_output=True) + tc_build.utils.create_gitignore(self.folders.install) + + def can_use_perf(self): + # Make sure perf is in the environment + if shutil.which('perf'): + try: + perf_cmd = [ + 'perf', 'record', + '--branch-filter', 'any,u', + '--event', 'cycles:u', + '--output', '/dev/null', + '--', 'sleep', '1', + ] # yapf: disable + subprocess.run(perf_cmd, capture_output=True, check=True) + except subprocess.CalledProcessError: + pass # Fallthrough to False below + else: + return True + + return False + + def check_dependencies(self): + deps = ['cmake', 'curl', 'git', 'ninja'] + for dep in deps: + if not shutil.which(dep): + raise RuntimeError(f"Dependency ('{dep}') could not be found!") + + def configure(self): + if not self.folders.build: + raise RuntimeError('No build folder set?') + if not self.folders.source: + raise RuntimeError('No source folder set?') + if not self.tools: + raise RuntimeError('No build tools set?') + if not self.projects: + raise RuntimeError('No projects set?') + if not self.targets: + raise RuntimeError('No targets set?') + + self.validate_targets() + + # yapf: disable + cmake_cmd = [ + 'cmake', + '-B', self.folders.build, + '-G', 'Ninja', + '-S', Path(self.folders.source, 'llvm'), + '-Wno-dev', + ] + # yapf: enable + if self.quiet_cmake: + cmake_cmd.append('--log-level=NOTICE') + + if self.ccache: + if shutil.which('ccache'): + self.cmake_defines['CMAKE_C_COMPILER_LAUNCHER'] = 'ccache' + self.cmake_defines['CMAKE_CXX_COMPILER_LAUNCHER'] = 'ccache' + else: + tc_build.utils.print_warning( + 'ccache requested but could not be found on your system, ignoring...') + + if self.tools.clang_tblgen: + self.cmake_defines['CLANG_TABLEGEN'] = self.tools.clang_tblgen + + if self.tools.ar: + self.cmake_defines['CMAKE_AR'] = self.tools.ar + if self.tools.ranlib: + self.cmake_defines['CMAKE_RANLIB'] = self.tools.ranlib + if 'CMAKE_BUILD_TYPE' not in self.cmake_defines: + self.cmake_defines['CMAKE_BUILD_TYPE'] = 'Release' + self.cmake_defines['CMAKE_C_COMPILER'] = self.tools.cc + self.cmake_defines['CMAKE_CXX_COMPILER'] = self.tools.cxx + if self.bolt: + self.cmake_defines['CMAKE_EXE_LINKER_FLAGS'] = '-Wl,--emit-relocs' + if self.folders.install: + self.cmake_defines['CMAKE_INSTALL_PREFIX'] = self.folders.install + + self.cmake_defines['LLVM_ENABLE_PROJECTS'] = ';'.join(self.projects) + # execinfo.h might not exist (Alpine Linux) but the GWP ASAN library + # depends on it. Disable the option to avoid breaking the build, the + # kernel does not depend on it. + if self.project_is_enabled('compiler-rt') and not Path('/usr/include/execinfo.h').exists(): + self.cmake_defines['COMPILER_RT_BUILD_GWP_ASAN'] = 'OFF' + if self.cmake_defines['CMAKE_BUILD_TYPE'] == 'Release': + self.cmake_defines['LLVM_ENABLE_WARNINGS'] = 'OFF' + if self.tools.llvm_tblgen: + self.cmake_defines['LLVM_TABLEGEN'] = self.tools.llvm_tblgen + self.cmake_defines['LLVM_TARGETS_TO_BUILD'] = ';'.join(self.targets) + if self.tools.ld: + self.cmake_defines['LLVM_USE_LINKER'] = self.tools.ld + + # Clear Linux needs a different target to find all of the C++ header files, otherwise + # stage 2+ compiles will fail without this + # We figure this out based on the existence of x86_64-generic-linux in the C++ headers path + if list(Path('/usr/include/c++').glob('*/x86_64-generic-linux')): + self.cmake_defines['LLVM_HOST_TRIPLE'] = 'x86_64-generic-linux' + + # By default, the Linux triples are for glibc, which might not work on + # musl-based systems. If clang is available, get the default target triple + # from it so that clang without a '--target' flag always works. This + # behavior can be opted out of by setting DISTRIBUTING=1 in the + # script's environment, in case the builder intends to distribute the + # toolchain, as this may not be portable. Since distribution is not a + # primary goal of tc-build, this is not abstracted further. + if shutil.which('clang') and not os.environ.get('DISTRIBUTING'): + default_target_triple = subprocess.run(['clang', '-print-target-triple'], + capture_output=True, + check=True, + text=True).stdout.strip() + self.cmake_defines['LLVM_DEFAULT_TARGET_TRIPLE'] = default_target_triple + + cmake_cmd += [f'-D{key}={self.cmake_defines[key]}' for key in sorted(self.cmake_defines)] + + self.clean_build_folder() + self.run_cmd(cmake_cmd) + + def host_target(self): + uname_to_llvm = { + 'aarch64': 'AArch64', + 'armv7l': 'ARM', + 'i386': 'X86', + 'mips': 'Mips', + 'mips64': 'Mips', + 'ppc': 'PowerPC', + 'ppc64': 'PowerPC', + 'ppc64le': 'PowerPC', + 'riscv32': 'RISCV', + 'riscv64': 'RISCV', + 's390x': 'SystemZ', + 'x86_64': 'X86', + } + return uname_to_llvm.get(platform.machine()) + + def host_target_is_enabled(self): + return 'all' in self.targets or self.host_target() in self.targets + + def project_is_enabled(self, project): + return 'all' in self.projects or project in self.projects + + def show_install_info(self): + # Installation folder is optional, show build folder as the + # installation location in that case. + install_folder = self.folders.install if self.folders.install else self.folders.build + if not install_folder: + raise RuntimeError('Installation folder not set?') + if not install_folder.exists(): + raise RuntimeError('Installation folder does not exist, run build()?') + if not (bin_folder := Path(install_folder, 'bin')).exists(): + raise RuntimeError('bin folder does not exist in installation folder, run build()?') + + tc_build.utils.print_header('LLVM installation information') + install_info = (f"Toolchain is available at: {install_folder}\n\n" + 'To use, either run:\n\n' + f"\t$ export PATH={bin_folder}:$PATH\n\n" + 'or add:\n\n' + f"\tPATH={bin_folder}:$PATH\n\n" + 'before the command you want to use this toolchain.\n') + print(install_info) + + for tool in ['clang', 'ld.lld']: + if (binary := Path(bin_folder, tool)).exists(): + subprocess.run([binary, '--version'], check=True) + print() + tc_build.utils.flush_std_err_out() + + def validate_targets(self): + if not self.folders.source: + raise RuntimeError('No source folder set?') + if not self.targets: + raise RuntimeError('No targets set?') + + all_targets = get_all_targets(self.folders.source) + + for target in self.targets: + if target in ('all', 'host'): + continue + + if target not in all_targets: + # tuple() for shorter pretty printing versus instead of + # ('{"', '".join(all_targets)}') + raise RuntimeError( + f"Requested target ('{target}') was not found in LLVM_ALL_TARGETS {tuple(all_targets)}, check spelling?" + ) + + +class LLVMSlimBuilder(LLVMBuilder): + + def configure(self): + # yapf: disable + slim_clang_defines = { + # Objective-C Automatic Reference Counting (we don't use Objective-C) + # https://clang.llvm.org/docs/AutomaticReferenceCounting.html + 'CLANG_ENABLE_ARCMT': 'OFF', + # We don't (currently) use the static analyzer and it saves cycles + # according to Chromium OS: + # https://crrev.com/44702077cc9b5185fc21e99485ee4f0507722f82 + 'CLANG_ENABLE_STATIC_ANALYZER': 'OFF', + # We don't use the plugin system and it will remove unused symbols: + # https://crbug.com/917404 + 'CLANG_PLUGIN_SUPPORT': 'OFF', + } + + slim_llvm_defines = { + # Don't build bindings; they are for other languages that the kernel does not use + 'LLVM_ENABLE_BINDINGS': 'OFF', + # Don't build Ocaml documentation + 'LLVM_ENABLE_OCAMLDOC': 'OFF', + # Don't build clang-tools-extras to cut down on build targets (about 400 files or so) + 'LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR': '', + # Don't include documentation build targets because it is available on the web + 'LLVM_INCLUDE_DOCS': 'OFF', + # Don't include example build targets to save on cmake cycles + 'LLVM_INCLUDE_EXAMPLES': 'OFF', + } + + slim_compiler_rt_defines = { + # Don't build libfuzzer when compiler-rt is enabled, it invokes cmake again and we don't use it + 'COMPILER_RT_BUILD_LIBFUZZER': 'OFF', + # We only use compiler-rt for the sanitizers, disable some extra stuff we don't need + # Chromium OS also does this: https://crrev.com/c/1629950 + 'COMPILER_RT_BUILD_CRT': 'OFF', + 'COMPILER_RT_BUILD_XRAY': 'OFF', + } + # yapf: enable + + self.cmake_defines.update(slim_llvm_defines) + if self.project_is_enabled('clang'): + self.cmake_defines.update(slim_clang_defines) + if self.project_is_enabled('compiler-rt') and self.cmake_defines.get( + 'LLVM_BUILD_RUNTIME', 'ON') == 'ON': + self.cmake_defines.update(slim_compiler_rt_defines) + + super().configure() + + +class LLVMBootstrapBuilder(LLVMSlimBuilder): + + def __init__(self): + super().__init__() + + self.projects = ['clang', 'lld'] + self.targets = ['host'] + + def configure(self): + if self.project_is_enabled('compiler-rt'): + self.cmake_defines['COMPILER_RT_BUILD_SANITIZERS'] = 'OFF' + + self.cmake_defines['CMAKE_BUILD_TYPE'] = 'Release' + self.cmake_defines['LLVM_BUILD_UTILS'] = 'OFF' + self.cmake_defines['LLVM_ENABLE_ASSERTIONS'] = 'OFF' + self.cmake_defines['LLVM_ENABLE_BACKTRACES'] = 'OFF' + self.cmake_defines['LLVM_INCLUDE_TESTS'] = 'OFF' + + super().configure() + + +class LLVMInstrumentedBuilder(LLVMBuilder): + + def __init__(self): + super().__init__() + + self.cmake_defines['LLVM_BUILD_INSTRUMENTED'] = 'IR' + self.cmake_defines['LLVM_BUILD_RUNTIME'] = 'OFF' + self.cmake_defines['LLVM_LINK_LLVM_DYLIB'] = 'ON' + + def configure(self): + # The following defines are needed to avoid thousands of warnings + # along the lines of: + # "Unable to track new values: Running out of static counters." + # They require LLVM_LINK_DYLIB to be enabled, which is done above. + cmake_options = Path(self.folders.source, 'llvm/cmake/modules/HandleLLVMOptions.cmake') + cmake_text = cmake_options.read_text(encoding='utf-8') + if 'LLVM_VP_COUNTERS_PER_SITE' in cmake_text: + self.cmake_defines['LLVM_VP_COUNTERS_PER_SITE'] = '6' + else: + cflags = [] + cxxflags = [] + + if 'CMAKE_C_FLAGS' in self.cmake_defines: + cflags += self.cmake_defines['CMAKE_C_FLAGS'].split(' ') + if 'CMAKE_CXX_FLAGS' in self.cmake_defines: + cxxflags += self.cmake_defines['CMAKE_CXX_FLAGS'].split(' ') + + vp_counters = [ + '-Xclang', + '-mllvm', + '-Xclang', + '-vp-counters-per-site=6', + ] + cflags += vp_counters + cxxflags += vp_counters + + self.cmake_defines['CMAKE_C_FLAGS'] = ' '.join(cflags) + self.cmake_defines['CMAKE_CXX_FLAGS'] = ' '.join(cxxflags) + + super().configure() + + def generate_profdata(self): + if not (profiles := list(self.folders.build.joinpath('profiles').glob('*.profraw'))): + raise RuntimeError('No profiles generated?') + + llvm_prof_data_cmd = [ + self.tools.llvm_profdata, + 'merge', + f"-output={Path(self.folders.build, 'profdata.prof')}", + *profiles, + ] + subprocess.run(llvm_prof_data_cmd, check=True) + + +class LLVMSlimInstrumentedBuilder(LLVMInstrumentedBuilder, LLVMSlimBuilder): + # No methods to override, this class inherits everyting from these super classes + pass + + +class LLVMSourceManager: + + def __init__(self, repo): + self.repo = repo + + def default_projects(self): + return ['clang', 'compiler-rt', 'lld', 'polly'] + + def default_targets(self): + all_targets = get_all_targets(self.repo) + targets = ['AArch64', 'ARM', 'BPF', 'Hexagon', 'Mips', 'PowerPC', 'RISCV', 'SystemZ', 'X86'] + + if 'LoongArch' in all_targets: + targets.append('LoongArch') + + return targets + + def download(self, ref, shallow=False): + if self.repo.exists(): + return + + tc_build.utils.print_header('Downloading LLVM') + + git_clone = ['git', 'clone'] + if shallow: + git_clone.append('--depth=1') + if ref != 'main': + git_clone.append('--no-single-branch') + git_clone += ['https://github.com/llvm/llvm-project', self.repo] + + subprocess.run(git_clone, check=True) + + self.git(['checkout', ref]) + + def git(self, cmd, capture_output=False): + return subprocess.run(['git', *cmd], + capture_output=capture_output, + check=True, + cwd=self.repo, + text=True) + + def git_capture(self, cmd): + return self.git(cmd, capture_output=True).stdout.strip() + + def is_shallow(self): + git_dir = self.git_capture(['rev-parse', '--git-dir']) + return Path(git_dir, 'shallow').exists() + + def ref_exists(self, ref): + try: + self.git(['show-branch', ref]) + except subprocess.CalledProcessError: + return False + return True + + def update(self, ref): + tc_build.utils.print_header('Updating LLVM') + + self.git(['fetch', 'origin']) + + if self.is_shallow() and not self.ref_exists(ref): + raise RuntimeError(f"Repo is shallow and supplied ref ('{ref}') does not exist!") + + self.git(['checkout', ref]) + + local_ref = None + with contextlib.suppress(subprocess.CalledProcessError): + local_ref = self.git_capture(['symbolic-ref', '-q', 'HEAD']) + if local_ref and local_ref.startswith('refs/heads/'): + self.git(['pull', '--rebase', 'origin', local_ref.replace('refs/heads/', '')]) diff --git a/tc_build/source.py b/tc_build/source.py new file mode 100644 index 0000000..4664421 --- /dev/null +++ b/tc_build/source.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import hashlib +import re +import subprocess + +import tc_build.utils + +# When doing verification, read 128MiB at a time +BYTES_TO_READ = 131072 + + +class Tarball: + + def __init__(self): + self.base_download_url = None + self.local_location = None + self.remote_tarball_name = None + self.remote_checksum_name = '' + + def download(self): + if not self.local_location: + raise RuntimeError('No local tarball location specified?') + if self.local_location.exists(): + return # Already downloaded + + if not self.base_download_url: + raise RuntimeError('No tarball download URL specified?') + if not self.remote_tarball_name: + self.remote_tarball_name = self.local_location.name + + tc_build.utils.curl(f"{self.base_download_url}/{self.remote_tarball_name}", + destination=self.local_location) + + # If there is a remote checksum file, download it, find the checksum + # for the particular tarball, compute the downloaded file's checksum, + # and finally compare the two. + if self.remote_checksum_name: + checksums = tc_build.utils.curl(f"{self.base_download_url}/{self.remote_checksum_name}") + if not (match := re.search( + fr"([0-9a-f]+)\s+{self.remote_tarball_name}$", checksums, flags=re.M)): + raise RuntimeError(f"Could not find checksum for {self.remote_tarball_name}?") + + if 'sha256' in self.remote_checksum_name: + file_hash = hashlib.sha256() + elif 'sha512' in self.remote_checksum_name: + file_hash = hashlib.sha512() + else: + raise RuntimeError( + f"No supported hashlib for {self.remote_checksum_name}, add support for it?") + with self.local_location.open('rb') as file: + while (data := file.read(BYTES_TO_READ)): + file_hash.update(data) + + computed_checksum = file_hash.hexdigest() + expected_checksum = match.groups()[0] + if computed_checksum != expected_checksum: + raise RuntimeError( + f"Computed checksum of {self.local_destination} ('{computed_checksum}') differs from expected checksum ('{expected_checksum}'), remove it and try again?" + ) + + def extract(self, extraction_location): + if not self.local_location: + raise RuntimeError('No local tarball location specified?') + if not self.local_location.exists(): + raise RuntimeError( + f"Local tarball ('{self.local_location}') could not be found, download it first?") + + extraction_location.mkdir(exist_ok=True, parents=True) + tar_cmd = [ + 'tar', + '--auto-compress', + f"--directory={extraction_location}", + '--extract', + f"--file={self.local_location}", + '--strip-components=1', + ] + + tc_build.utils.print_info(f"Extracting {self.local_location} into {extraction_location}...") + subprocess.run(tar_cmd, check=True) + + +class SourceManager: + + def __init__(self, location=None): + self.location = location + self.tarball = Tarball() diff --git a/tc_build/tools.py b/tc_build/tools.py new file mode 100644 index 0000000..2d179b7 --- /dev/null +++ b/tc_build/tools.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# pylint: disable=invalid-name + +import os +from pathlib import Path +import re +import shutil +import subprocess + +import tc_build.utils + + +class HostTools: + + def __init__(self): + self.cc = self.find_host_cc() + self.cc_is_clang = 'clang' in self.cc.name + + self.ar = self.find_host_ar() + self.cxx = self.find_host_cxx() + self.ld = self.find_host_ld() + self.ranlib = self.find_host_ranlib() + + self.clang_tblgen = None + self.llvm_bolt = None + self.llvm_profdata = None + self.llvm_tblgen = None + self.merge_fdata = None + self.perf2bolt = None + + def find_host_ar(self): + # GNU ar is the default, no need for llvm-ar if using GCC + if not self.cc_is_clang: + return None + + if (ar := Path(self.cc.parent, 'llvm-ar')).exists(): + return ar + + return None + + def find_host_cc(self): + # resolve() is called here and below to get /usr/lib/llvm-#/bin/... for + # versioned LLVM binaries on Debian and Ubuntu. + if (cc := self.from_env('CC')): + return cc.resolve() + + possible_c_compilers = [*self.generate_versioned_binaries(), 'clang', 'gcc'] + for compiler in possible_c_compilers: + if (cc := shutil.which(compiler)): + break + + if not cc: + raise RuntimeError('Neither clang nor gcc could be found on your system?') + + return Path(cc).resolve() # resolve() for Debian/Ubuntu variants + + def find_host_cxx(self): + if (cxx := self.from_env('CXX')): + return cxx + + possible_cxx_compiler = 'clang++' if self.cc_is_clang else 'g++' + + # Use CXX from the 'bin' folder of CC if it exists + if (cxx := Path(self.cc.parent, possible_cxx_compiler)).exists(): + return cxx + + if not (cxx := shutil.which(possible_cxx_compiler)): + raise RuntimeError( + f"CXX ('{possible_cxx_compiler}') could not be found on your system?") + + return Path(cxx) + + def find_host_ld(self): + if (ld := self.from_env('LD')): + return ld + + if self.cc_is_clang: + # First, see if there is an ld.lld installed in the same folder as + # CC; if so, we know it can be used. + if (ld := Path(self.cc.parent, 'ld.lld')).exists(): + return ld + + # If not, try to find a suitable linker via PATH + possible_linkers = ['lld', 'gold', 'bfd'] + for linker in possible_linkers: + if (ld := shutil.which(f"ld.{linker}")): + break + if not ld: + return None + return self.validate_ld(Path(ld)) + + # For GCC, it is only worth testing 'gold' + return self.validate_ld('gold') + + def find_host_ranlib(self): + # GNU ranlib is the default, no need for llvm-ranlib if using GCC + if not self.cc_is_clang: + return None + + if (ranlib := Path(self.cc.parent, 'llvm-ranlib')).exists(): + return ranlib + + return None + + def from_env(self, key): + if key not in os.environ: + return None + + if key == 'LD': + return self.validate_ld(os.environ[key], warn=True) + + if not (tool := shutil.which(os.environ[key])): + raise RuntimeError( + f"{key} value ('{os.environ[key]}') could not be found on your system?") + return Path(tool) + + def generate_versioned_binaries(self): + try: + cmakelists_txt = tc_build.utils.curl( + 'https://raw.githubusercontent.com/llvm/llvm-project/main/llvm/CMakeLists.txt') + except subprocess.CalledProcessError: + llvm_tot_ver = 16 + else: + if not (match := re.search(r'set\(LLVM_VERSION_MAJOR\s+(\d+)', cmakelists_txt)): + raise RuntimeError('Could not find LLVM_VERSION_MAJOR in CMakeLists.txt?') + llvm_tot_ver = int(match.groups()[0]) + + return [f'clang-{num}' for num in range(llvm_tot_ver, 6, -1)] + + def show_compiler_linker(self): + print(f"CC: {self.cc}") + print(f"CXX: {self.cxx}") + if self.ld: + if isinstance(self.ld, Path): + print(f"LD: {self.ld}") + else: + ld_to_print = self.ld if 'ld.' in self.ld else f"ld.{self.ld}" + print(f"LD: {shutil.which(ld_to_print)}") + tc_build.utils.flush_std_err_out() + + def validate_ld(self, ld, warn=False): + if not ld: + return None + + cc_cmd = [self.cc, f'-fuse-ld={ld}', '-o', '/dev/null', '-x', 'c', '-'] + try: + subprocess.run(cc_cmd, + capture_output=True, + check=True, + input='int main(void) { return 0; }', + text=True) + except subprocess.CalledProcessError: + if warn: + tc_build.utils.print_warning( + f"LD value ('{ld}') is not supported by CC ('{self.cc}'), ignoring it...") + return None + + return ld + + +class StageTools: + + def __init__(self, bin_folder): + # Used by cmake + self.ar = Path(bin_folder, 'llvm-ar') + self.cc = Path(bin_folder, 'clang') + self.clang_tblgen = Path(bin_folder, 'clang-tblgen') + self.cxx = Path(bin_folder, 'clang++') + self.ld = Path(bin_folder, 'ld.lld') + self.llvm_tblgen = Path(bin_folder, 'llvm-tblgen') + self.ranlib = Path(bin_folder, 'llvm-ranlib') + # Used by the builder + self.llvm_bolt = Path(bin_folder, 'llvm-bolt') + self.llvm_profdata = Path(bin_folder, 'llvm-profdata') + self.merge_fdata = Path(bin_folder, 'merge-fdata') + self.perf2bolt = Path(bin_folder, 'perf2bolt') diff --git a/tc_build/utils.py b/tc_build/utils.py new file mode 100644 index 0000000..0725299 --- /dev/null +++ b/tc_build/utils.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import time + + +def create_gitignore(folder): + folder.joinpath('.gitignore').write_text('*\n', encoding='utf-8') + + +def curl(url, capture_output=True, destination=None, text=True): + curl_cmd = ['curl', '-fLSs'] + if destination: + curl_cmd += ['-o', destination] + curl_cmd.append(url) + return subprocess.run(curl_cmd, capture_output=capture_output, check=True, text=text).stdout + + +def flush_std_err_out(): + sys.stderr.flush() + sys.stdout.flush() + + +def get_duration(start_seconds, end_seconds=None): + if not end_seconds: + end_seconds = time.time() + seconds = int(end_seconds - start_seconds) + days, seconds = divmod(seconds, 60 * 60 * 24) + hours, seconds = divmod(seconds, 60 * 60) + minutes, seconds = divmod(seconds, 60) + + parts = [] + if days: + parts.append(f"{days}d") + if hours: + parts.append(f"{hours}h") + if minutes: + parts.append(f"{minutes}m") + parts.append(f"{seconds}s") + + return ' '.join(parts) + + +def libc_is_musl(): + # musl's ldd does not appear to support '--version' directly, as its return + # code is 1 and it prints all text to stderr. However, it does print the + # version information so it is good enough. Just 'check=False' it and move + # on. + ldd_out = subprocess.run(['ldd', '--version'], capture_output=True, check=False, text=True) + return 'musl' in (ldd_out.stderr if ldd_out.stderr else ldd_out.stdout) + + +def print_color(color, string): + print(f"{color}{string}\033[0m", flush=True) + + +def print_cyan(msg): + print_color('\033[01;36m', msg) + + +def print_header(string): + border = ''.join(["=" for _ in range(len(string) + 6)]) + print_cyan(f"\n{border}\n== {string} ==\n{border}\n") + + +def print_info(msg): + print(f"I: {msg}", flush=True) + + +def print_warning(msg): + print_color('\033[01;33m', f"W: {msg}") diff --git a/utils.py b/utils.py deleted file mode 100755 index d962511..0000000 --- a/utils.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -# Description: Common helper functions - -import hashlib -import pathlib -import shutil -import subprocess - - -def create_gitignore(folder): - """ - Create a gitignore that ignores all files in a folder. Some folders are not - known until the script is run so they can't be added to the root .gitignore - :param folder: Folder to create the gitignore in - """ - with folder.joinpath(".gitignore").open("w") as gitignore: - gitignore.write("*") - - -def current_binutils(): - """ - Simple getter for current stable binutils release - :return: The current stable release of binutils - """ - return "binutils-2.38" - - -def download_binutils(folder): - """ - Downloads the latest stable version of binutils - :param folder: Directory to download binutils to - """ - binutils = current_binutils() - binutils_folder = folder.joinpath(binutils) - if not binutils_folder.is_dir(): - # Remove any previous copies of binutils - for entity in folder.glob('binutils-*'): - if entity.is_dir(): - shutil.rmtree(entity.as_posix()) - else: - entity.unlink() - - # Download the tarball - binutils_tarball = folder.joinpath(binutils + ".tar.xz") - subprocess.run([ - "curl", "-LSs", "-o", - binutils_tarball.as_posix(), - "https://ftp.gnu.org/gnu/binutils/" + binutils_tarball.name - ], - check=True) - verify_binutils_checksum(binutils_tarball) - # Extract the tarball then remove it - subprocess.run(["tar", "-xJf", binutils_tarball.name], - check=True, - cwd=folder.as_posix()) - create_gitignore(binutils_folder) - binutils_tarball.unlink() - - -def verify_binutils_checksum(file): - # Check the SHA512 checksum of the downloaded file with a known good one - # The sha512.sum file from ships the SHA512 checksums - # Link: https://sourceware.org/pub/binutils/releases/sha512.sum - file_hash = hashlib.sha512() - with file.open("rb") as f: - while True: - data = f.read(131072) - if not data: - break - file_hash.update(data) - good_hash = "8bf0b0d193c9c010e0518ee2b2e5a830898af206510992483b427477ed178396cd210235e85fd7bd99a96fc6d5eedbeccbd48317a10f752b7336ada8b2bb826d" - if file_hash.hexdigest() != good_hash: - raise RuntimeError( - "binutils: SHA512 checksum does not match known good one!") - - -def print_header(string): - """ - Prints a fancy header - :param string: String to print inside the header - """ - # Use bold cyan for the header so that the headers - # are not intepreted as success (green) or failed (red) - print("\033[01;36m") - for x in range(0, len(string) + 6): - print("=", end="") - print("\n== %s ==" % string) - for x in range(0, len(string) + 6): - print("=", end="") - # \033[0m resets the color back to the user's default - print("\n\033[0m") - - -def print_error(string): - """ - Prints a error in bold red - :param string: String to print - """ - # Use bold red for error - print("\033[01;31m%s\n\033[0m" % string) - - -def print_warning(string): - """ - Prints a error in bold yellow - :param string: String to print - """ - # Use bold yellow for error - print("\033[01;33m%s\n\033[0m" % string)