Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: use cache kernel build output #694

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: use cache kernel build output #694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions drbd/pkg.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,27 @@ dependencies:
- stage: kernel-build
steps:
- sources:
- url: https://pkg.linbit.com//downloads/drbd/{{ regexReplaceAll ".\\d+\\.\\d+$" .drbd_version "${1}" }}/drbd-{{ .drbd_version }}.tar.gz
- url: https://pkg.linbit.com/downloads/drbd/{{ regexReplaceAll ".\\d+\\.\\d+$" .drbd_version "${1}" }}/drbd-{{ .drbd_version }}.tar.gz
destination: drbd.tar.gz
sha256: "{{ .drbd_sha256 }}"
sha512: "{{ .drbd_sha512 }}"
env:
ARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}x86_64{{ else }}unsupported{{ end }}
cache: /src/build
prepare:
- |
tar -xzf drbd.tar.gz --strip-components=1
build:
- |
make -j $(nproc) -C drbd KERNEL_SOURCES=/src MODVERSIONS=detect KDIR=/src
make -j $(nproc) -C drbd KERNEL_SOURCES=/src MODVERSIONS=detect KDIR=/src/build
install:
- |
mkdir -p /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.order /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
mkdir -p /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.order /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/

make -j $(nproc) -C /src M=$(pwd)/drbd modules_install DESTDIR=/rootfs INSTALL_MOD_PATH=/rootfs INSTALL_MOD_DIR=extras INSTALL_MOD_STRIP=1 CONFIG_MODULE_SIG_ALL=y
make -j $(nproc) -C /src/build M=$(pwd)/drbd modules_install DESTDIR=/rootfs INSTALL_MOD_PATH=/rootfs INSTALL_MOD_DIR=extras INSTALL_MOD_STRIP=1 CONFIG_MODULE_SIG_ALL=y
test:
- |
# https://www.kernel.org/doc/html/v4.15/admin-guide/module-signing.html#signed-modules-and-stripping
Expand Down
15 changes: 8 additions & 7 deletions gasket-driver/pkg.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,24 @@ steps:
sha512: "{{ .gasket_driver_sha512 }}"
env:
ARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}x86_64{{ else }}unsupported{{ end }}
cache: /src/build
prepare:
- |
tar xf gasket-driver.tar.gz --strip-components=1
build:
- |
cd src
sed -i 's|/lib/modules/$(KVERSION)/build|/src|' ./Makefile
make all
sed -i 's|/lib/modules/$(KVERSION)/build|/src/build|' ./Makefile
make -j $(nproc) all

install:
- |
mkdir -p /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.order /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
mkdir -p /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.order /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/

make -C /src M=$(pwd)/src modules_install INSTALL_MOD_PATH=/rootfs INSTALL_MOD_DIR=extras INSTALL_MOD_STRIP=1
make -j $(nproc) -C /src/build M=$(pwd)/src modules_install INSTALL_MOD_PATH=/rootfs INSTALL_MOD_DIR=extras INSTALL_MOD_STRIP=1
test:
- |
# https://www.kernel.org/doc/html/v4.15/admin-guide/module-signing.html#signed-modules-and-stripping
Expand Down
6 changes: 4 additions & 2 deletions kernel/build/pkg.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@ steps:
- env:
CARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}amd64{{ else }}unsupported{{ end }}
ARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}x86_64{{ else }}unsupported{{ end }}
KBUILD_OUTPUT: /src/build
cache: /src/build
prepare:
- |
cd /src
cp -v /pkg/config-${CARCH} .config
cp -v /pkg/config-${CARCH} ${KBUILD_OUTPUT}/.config
cp -v /pkg/certs/* certs/
build:
- |
cd /src
python3 /toolchain/kconfig-hardened-check/bin/kconfig-hardened-check -c .config -m json | python3 /pkg/scripts/filter-hardened-check.py
python3 /toolchain/kconfig-hardened-check/bin/kconfig-hardened-check -c ${KBUILD_OUTPUT}/.config -m json | python3 /pkg/scripts/filter-hardened-check.py
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

my concern is how can we be sure that the cache gets properly invalidated? I guess in default make, it's all timestamp-based, so if we have two kernel builds from different branches, it may very well poison the cache?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The src folder gets changed, but the kernel build objects can still be re-used. I don;t want this going in release-1.4 anyways, targetting for 1.5

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, I understand that, but my point is the following, the Makefile is something like:

/src/build/foo.o: /src/foo.c
  cc -o /src/build/foo.o /src/foo.c

So what make does is that it compares the timestamps of .c and .o files: if .o is newer, it skips the build

But this is not reliable enough given that we have multiple branches/versions of the kernel, and timestamp for Linux 5.15.x and 6.1.x might be inverted.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the kernel build is intelligent enough to detect that, based on the source. At least when building locally, but I'ven't tried changing versions 😅 . I'll test locally later

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if there's some smartness, let's find some proof on that :) it's definitely not in make itself

- |
cd /src

Expand Down
7 changes: 4 additions & 3 deletions kernel/kernel/pkg.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ dependencies:
steps:
- env:
ARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}x86_64{{ else }}unsupported{{ end }}
cache: /src/build
install:
- |
cd /src
cd /src/build

mkdir -p /rootfs/boot
mkdir -p /rootfs/dtb
case $ARCH in
x86_64)
mv arch/x86/boot/bzImage /rootfs/boot/vmlinuz
cp arch/x86/boot/bzImage /rootfs/boot/vmlinuz
;;
arm64)
mv arch/arm64/boot/Image /rootfs/boot/vmlinuz
cp arch/arm64/boot/Image /rootfs/boot/vmlinuz
cd ./arch/arm64/boot/dts
for vendor in $(find . -not -path . -type d); do
dest="/rootfs/dtb/$vendor"
Expand Down
13 changes: 7 additions & 6 deletions nonfree/kmod-nvidia/pkg.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ steps:
# {{ end }} This in fact is YAML comment, but Go templating instruction is evaluated by bldr
env:
ARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}x86_64{{ else }}unsupported{{ end }}
cache: /src/build
prepare:
- |
export PATH=/toolchain/bin:$PATH
Expand All @@ -31,17 +32,17 @@ steps:
- |
cd NVIDIA-Linux-*/kernel

make -j $(nproc) SYSSRC=/src
make -j $(nproc) SYSSRC=/src SYSOUT=/src/build
install:
- |
cd NVIDIA-Linux-*/kernel

mkdir -p /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.order /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
mkdir -p /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.order /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/

make -j $(nproc) modules_install SYSSRC=/src DEPMOD=/toolchain/bin/depmod INSTALL_MOD_PATH=/rootfs INSTALL_MOD_DIR=extras INSTALL_MOD_STRIP=1
make -j $(nproc) modules_install SYSSRC=/src SYSOUT=/src/build DEPMOD=/toolchain/bin/depmod INSTALL_MOD_PATH=/rootfs INSTALL_MOD_DIR=extras INSTALL_MOD_STRIP=1
finalize:
- from: /rootfs
to: /
13 changes: 7 additions & 6 deletions nvidia-open-gpu-kernel-modules/pkg.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ steps:
destination: open-gpu-kernel-modules.tar.gz
sha256: "{{ .nvidia_driver_sha256 }}"
sha512: "{{ .nvidia_driver_sha512 }}"
cache: /src/build
env:
ARCH: {{ if eq .ARCH "aarch64"}}arm64{{ else if eq .ARCH "x86_64" }}x86_64{{ else }}unsupported{{ end }}
prepare:
Expand All @@ -22,15 +23,15 @@ steps:
tar xf open-gpu-kernel-modules.tar.gz --strip-components=1
build:
- |
make -j $(nproc) SYSSRC=/src
make -j $(nproc) SYSSRC=/src SYSOUT=/src/build
install:
- |
mkdir -p /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.order /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
cp /src/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/include/config/kernel.release)/
mkdir -p /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.order /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/
cp /src/build/modules.builtin.modinfo /rootfs/lib/modules/$(cat /src/build/include/config/kernel.release)/

make -j $(nproc) modules_install SYSSRC=/src DEPMOD=/toolchain/bin/depmod INSTALL_MOD_PATH=/rootfs INSTALL_MOD_STRIP=1
make -j $(nproc) modules_install SYSSRC=/src SYSOUT=/src/build DEPMOD=/toolchain/bin/depmod INSTALL_MOD_PATH=/rootfs INSTALL_MOD_STRIP=1
test:
- |
# https://www.kernel.org/doc/html/v4.15/admin-guide/module-signing.html#signed-modules-and-stripping
Expand Down