buildmaster: Rework container to be less complex

* Haikuporter is now packaged with container
* Haiku host-tools now packaged with container
* Reduce the amount of one-off on-disk config files
* Upgrade to Debian bullseye-slim

.dockerignore

@@ -0,0 +1 @@
+**/.git

buildmaster/backend/Dockerfile

@@ -0,0 +1,50 @@
+FROM debian:bullseye-slim AS host-tools
+
+RUN apt-get update \
+	&& apt-get -y install git bc nasm texinfo flex bison gawk build-essential \
+		unzip wget zip less zlib1g-dev libzstd-dev python3
+
+# We can skip buildtools someday if we ever get a way to build jam without it
+RUN git clone --depth 1 https://review.haiku-os.org/buildtools /tmp/buildtools \
+	&& git clone --depth 1 https://review.haiku-os.org/haiku /tmp/haiku \
+	&& cd /tmp/buildtools/jam && make && ./jam0 install \
+	&& cd /tmp/haiku && ./configure --host-only \
+	&& jam -j2 -q \<build\>package \<build\>package_repo
+
+#############################################################
+
+FROM debian:bullseye-slim
+
+# Pre-requirements
+RUN apt-get update \
+	&& apt-get -y install attr autoconf automake bison coreutils curl flex \
+		gawk gcc gcc-multilib g++ git libcurl4-openssl-dev make nasm python3 \
+		python3-paramiko python3-pip tar texinfo wget zlib1g-dev \
+	&& apt-get clean \
+	&& echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib' >> /etc/bash.bashrc
+
+# Minisign for repo signatures
+RUN wget https://github.com/jedisct1/minisign/releases/download/0.10/minisign-0.10-linux.tar.gz -O /tmp/minisign.tar.gz \
+	&& cd /tmp && tar -xvz --strip=2 -f /tmp/minisign.tar.gz && mv minisign /usr/local/bin \
+	&& chmod 755 /usr/local/bin/minisign
+
+# Haikuporter from local context root (this is where the weird context requirement comes from)
+ADD . /tmp/haikuporter
+RUN pip3 install /tmp/haikuporter \
+	&& cp /tmp/haikuporter/buildmaster/backend/assets/bin/* /usr/local/bin/ \
+	&& rm -rf /tmp/*
+
+# Haikuporter release (another option, but weird from within haikuporter repo)
+#ENV HAIKUPORTER_VERSION="1.2.6"
+#RUN pip3 install https://github.com/haikuports/haikuporter/archive/refs/tags/$HAIKUPORTER_VERSION.tar.gz
+
+COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package/package /usr/local/bin/
+COPY --from=host-tools /tmp/haiku/generated/objects/linux/x86_64/release/tools/package_repo/package_repo /usr/local/bin/
+COPY --from=host-tools /tmp/haiku/generated/objects/linux/lib/* /usr/local/lib/
+
+RUN mkdir /var/sources /var/packages /var/buildmaster
+
+VOLUME ["/var/sources", "/var/packages", "/var/buildmaster"]
+WORKDIR /var/buildmaster
+
+COPY buildmaster/backend/assets/bootstrap buildmaster/backend/assets/loop /bin/

buildmaster/backend/Makefile

@@ -0,0 +1,9 @@
+#
+# CAREFUL HERE! The build context is the top level directory of git!
+#
+default:
+	docker build --no-cache --tag docker.io/haikuporter/buildmaster:1.2.6-1 -f Dockerfile ../..
+push:
+	docker push docker.io/haikuporter/buildmaster:1.2.6-1
+enter:
+	docker run -it docker.io/haikuporter/buildmaster:1.2.6-1 /bin/bash -l

buildmaster/backend/README.md

@@ -0,0 +1,31 @@
+# HaikuPorter in Buildmaster mode
+
+One buildmaster container per architecture
+
+# Requirements
+
+## Secrets
+
+  * ```/run/secrets/sig_repo_privatekey``` - Minisign private key to sign repos (optional)
+  * ```/run/secrets/sig_repo_privatekeypass``` - Password for Minisign private key (optional)
+
+## Environmental
+
+  * ```BUILD_TARGET_ARCH``` - Target architecture for buildmaster
+  * ```REPOSITORY_TRIGGER_URL``` - Target URL to hit when build complete (optional)
+    * example: https://depot.haiku-os.org/__repository/haikuports/source/haikuports_x86_64/import
+
+## Volumes
+
+  * /var/sources
+    * Storage for various required sources like haikuports or haiku
+  * /var/packages
+    * Storage for packages (TODO, more info)
+  * /var/buildmaster
+    * Main state directory for buildmaster
+    * haikuports
+      * buildmaster
+        * builders
+        * haikuports.conf
+    * package_tools
+      * host tools and libraries for managing packages

buildmaster/bin/buildmaster → buildmaster/backend/assets/bin/buildmaster

@@ -6,21 +6,15 @@ then
 	exit 1
 fi
 
-BASE_DIR="$(pwd)/buildmaster"
-CONFIG_FILE="$BASE_DIR/config"
-source "$CONFIG_FILE"
-
-if [ $? -ne 0 ]
+if [ -z "$BUILD_TARGET_ARCH" ]
 then
-	echo "configuration file $CONFIG_FILE couldn't be sourced"
+	echo "buildmaster instance configuration invalid"
 	exit 1
 fi
 
-if [ -z "$HAIKUPORTER" ]
-then
-	echo "HAIKUPORTER environment variable not set"
-	exit 1
-fi
+export BASE_DIR="$(pwd)/buildmaster"
+export REPO_DIR="/var/packages/repository/master/$BUILD_TARGET_ARCH/current"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/buildmaster/package_tools"
 
 case "$1" in
 	update)
@@ -42,11 +36,11 @@ case "$1" in
 		fi
 
 		echo "moving from $PREVIOUS_REVISION to $HEAD_REVISION"
-		"$HAIKUPORTER" --no-package-obsoletion --repository-update
+		haikuporter --no-package-obsoletion --repository-update
 
 		PORTS_TO_BUILD=$(git diff-tree -z -r --name-only --diff-filter ACMTR \
 				$PREVIOUS_REVISION..$HEAD_REVISION \
-			| xargs --null "$HAIKUPORTER" --no-package-obsoletion \
+			| xargs --null haikuporter --no-package-obsoletion \
 				--no-repository-update --ports-for-files \
 				--active-versions-only 2> /dev/null \
 			| sort -u)
@@ -59,7 +53,7 @@ case "$1" in
 		fi
 	;;
 	everything)
-		PORTS_TO_BUILD=$("$HAIKUPORTER" --no-package-obsoletion --print-raw \
+		PORTS_TO_BUILD=$(haikuporter --no-package-obsoletion --print-raw \
 			--list 2> /dev/null)
 	;;
 	build)
@@ -114,7 +108,7 @@ echo "$BUILDRUN" > "$BUILDRUN_FILE"
 rm "$BUILDRUN_BASE/current"
 ln -rs "$BUILDRUN_OUTPUT_DIR" "$BUILDRUN_BASE/current"
 
-"$HAIKUPORTER" --debug --build-master-output-dir="$BUILDRUN_OUTPUT_DIR" \
+haikuporter --debug --build-master-output-dir="$BUILDRUN_OUTPUT_DIR" \
 	--system-packages-directory="$BASE_DIR/initial-packages" \
 	--build-master $PORTS_TO_BUILD
 

buildmaster/backend/assets/bin/generate_initial_packages.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+if [ $# -ne 3 ]; then	
+	echo "generates an archive of initial_packages for haikuporter buildmaster"
+	echo "reminder: ensure haiku and buildtools directories are on the branches you want"
+	echo "usage: $0 <haiku dir> <buildtools dir> <arch>"
+	exit 1
+fi
+
+HAIKU_SRC=$(realpath $1)
+HAIKU_BRANCH=$(git -C "${HAIKU_SRC}" branch --show-current)
+BUILDTOOLS_SRC=$(realpath $2)
+BUILDTOOLS_BRANCH=$(git -C "${BUILDTOOLS_SRC}" branch --show-current)
+ARCH=$3
+WORK=~/.tmp/generated.$ARCH
+
+rm -rf ${WORK}
+mkdir -p ${WORK}
+
+if [ "${HAIKU_BRANCH}" == "master" ]; then
+	echo "Warning: haiku is the master branch, you likely don't want this!"
+fi
+if [ "${BUILDTOOLS_BRANCH}" == "master" ]; then
+	echo "Warning: buildtools is the master branch, you likely don't want this!"
+fi
+
+CPUS=$(nproc)
+if [ $CPUS -gt 8 ]; then
+	# a little cautious for parallel job bugs in our jam
+	CPUS=8
+fi
+
+CONFIGURE="$HAIKU_SRC/configure -j$CPUS --distro-compatibility official --cross-tools-source $BUILDTOOLS_SRC"
+
+if [ "${ARCH}" == "x86_gcc2h" ]; then
+	CONFIGURE="$CONFIGURE --build-cross-tools x86_gcc2 --build-cross-tools x86"
+else
+	CONFIGURE="$CONFIGURE --build-cross-tools $ARCH"
+fi
+
+## Build jam
+cd "$BUILDTOOLS_SRC"/jam
+make
+cp bin.*/jam $WORK
+
+cd $WORK
+
+# Configure
+$CONFIGURE
+
+# nightly-raw == golidlocks of packages
+./jam -q -j$CPUS @nightly-raw
+
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$WORK/objects/linux/lib"
+TOOLS="$WORK/objects/linux/$(uname -m)/release/tools/"
+
+rm -rf ~/.tmp/"haiku-${HAIKU_BRANCH}-$ARCH"
+mkdir -p ~/.tmp/"haiku-${HAIKU_BRANCH}-$ARCH";
+cd ~/.tmp/"haiku-${HAIKU_BRANCH}-$ARCH"
+
+for PACKAGE in "${WORK}"/objects/haiku/*/packaging/packages/*.hpkg
+do
+	cp "$PACKAGE" $("$TOOLS/package/package" info -f "%fileName%" "$PACKAGE")
+done
+cp "$WORK"/download/*.hpkg .
+
+cd ~/.tmp
+tar cvzf ~/.tmp/haiku-${HAIKU_BRANCH}-$ARCH.tar.gz haiku-${HAIKU_BRANCH}-$ARCH
+
+echo "Done!  ~/.tmp/haiku-${HAIKU_BRANCH}-$ARCH.tar.gz is ready for the Haiku buildmaster"

buildmaster/backend/assets/bootstrap

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+cd /var/sources
+
+git clone --depth=1 https://github.com/haikuports/haikuporter
+
+bootstrap_buildmaster --haikuporter-dir haikuporter --base-dir /var/buildmaster $@

buildmaster/docker/loop → buildmaster/backend/assets/loop

@@ -8,35 +8,34 @@ then
 	exit 1
 fi
 
-cd "$WORKDIR"
-
-. buildmaster/config
-
-if [ -z "$HAIKUPORTER" ]
+if [ -z "$BUILD_TARGET_ARCH" ]
 then
 	echo "buildmaster instance configuration invalid"
 	exit 1
 fi
 
+cd "$WORKDIR"
+
 SUCCESS_WAIT=60
 ERROR_WAIT=60
-COMMANDS_DIR="$(realpath "$(dirname "$HAIKUPORTER")/buildmaster/bin")"
-BUILDMASTER="$COMMANDS_DIR/buildmaster"
 
 export PYTHONUNBUFFERED=1
+export BUILD_TARGET_ARCH="$BUILD_TARGET_ARCH"
+export REPO_DIR="/var/packages/repository/master/$BUILD_TARGET_ARCH/current"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/buildmaster/package_tools"
 
 exec 2>&1
 
 build_prep () {
 	CONSISTENCY_REPORT_FILE="$REPO_DIR/repo_consistency.txt"
 	echo "repo consistency report at $(git rev-parse HEAD)" \
 		> "$CONSISTENCY_REPORT_FILE"
-	$HAIKUPORTER --debug --check-repository-consistency \
+	haikuporter --debug --check-repository-consistency \
 		--no-package-obsoletion \
 		--system-packages-directory buildmaster/initial-packages \
 		>> "$CONSISTENCY_REPORT_FILE"
 
-	$HAIKUPORTER --debug --prune-package-repository
+	haikuporter --debug --prune-package-repository
 }
 
 while true
@@ -52,7 +51,7 @@ do
 		rm buildmaster/do-everything
 
 		build_prep
-		$BUILDMASTER everything
+		buildmaster everything
 	elif [ -f buildmaster/do-packages ]; then
 		LIST=$(cat buildmaster/do-packages | tr '\n' ' ')
 		echo "$(date) buildmaster list requested, starting the following:"
@@ -61,7 +60,7 @@ do
 		rm buildmaster/do-packages
 
 		build_prep
-		$BUILDMASTER build $LIST
+		buildmaster build $LIST
 	elif [ $(git rev-parse HEAD) = $HEAD_REVISION ]; then
 		echo "$(date) no new revision"
 		sleep $SUCCESS_WAIT
@@ -71,7 +70,7 @@ do
 		echo ""
 
 		build_prep
-		$BUILDMASTER update
+		buildmaster update
 	fi
 	RESULT=$?
 
@@ -85,7 +84,8 @@ do
 
 	# Careful here
 	SIGFLAGS=""
-	if [ -f /run/secrets/sig_repo_privatekey && -f /run/secrets/sig_repo_privatekeypass ]; then
+	if [ -f /run/secrets/sig_repo_privatekey ] && [ -f /run/secrets/sig_repo_privatekeypass ]
+	then
 		echo "Info: Found package repository signature information."
 		touch /tmp/haiku-secret.key && chown 600 /tmp/haiku-secret.key
 		echo "untrusted comment: minisign encrypted secret key" > /tmp/haiku-secret.key
@@ -99,7 +99,7 @@ do
 
 	echo ""
 	echo "$(date) pruning packages and creating repository"
-	$HAIKUPORTER --debug --prune-package-repository \
+	haikuporter --debug --prune-package-repository \
 		--system-packages-directory buildmaster/initial-packages \
 		--check-package-repository-consistency \
 		--create-package-repository "$REPO_DIR" \