old-fashioned-image-build

#!/bin/bash

# This should be put in your PATH and then invoked (as root) from a checkout
# of livecd-rootfs that you'd like to build.

# Original user-data script by rcj, weaponized by tribaal

# TODO: Reuse buildd environment if found.

IMG_FORMAT="ext4"
PROJECT="ubuntu-cpc"
SUBPROJECT=""
PROPOSED_BUILD=""
PROPOSED_IN_IMAGE="false"
SERIES=""
USE_CHROOT_CACHE=false
CLEANUP=true
IMAGE_TARGET=""
REPO_SNAPSHOT=""
SNAP_COHORT=""
EXTRA_PPA=""
DEBUG=""

while :; do
    case "$1" in
        --minimized)
            # Pass this flag to build the 'minimal' image family
            SUBPROJECT="--subproject minimized"
            ;;
        --debug)
            DEBUG="--debug"
            ;;
        --build-from-proposed)
            # Build from -proposed pocket
            PROPOSED_BUILD="--proposed"
            ;;
        --enable-proposed-in-image)
            # Enable the -proposed pocket in the image(s) produced from the
            # build
            PROPOSED_IN_IMAGE="true"
            ;;
        --subproject)
            # Depends on project
            # Subprojects are handled by scripts in 
            #   livecd-rootfs/live-build/auto/, as well as by individual hooks
            # e.g. 'buildd'
            SUBPROJECT="--subproject $2"
            shift
            ;;
        --extra-ppa)
            # inject public ppas into the build environment
            # e.g. 'mylpname/someppa'
            EXTRA_PPA="--extra-ppa $2"
            shift
            ;;
        --image-format|--img_format)
            # Corresponds to the '--image-format' option for livecd-rootfs
            # e.g. 'ext4'
            IMG_FORMAT="$2"
            shift
            ;;
        --project)
            # Default for Old-Fashioned is 'ubuntu-cpc'
            # Corresponds to the '--project' option for livecd-rootfs
            # e.g. 'ubuntu-cpc' or 'ubuntu-base'
            if [ "$2" ]; then
                PROJECT="$2"
                shift
            fi
            ;;
        --series)
            # The name of the series/suite to build
            # e.g. 'bionic'
            if [ "$2" ]; then
                SERIES="$2"
                shift
            else
                echo "ERROR: --series requires a non-empty argument."
                exit 1
            fi
            ;;
        --use-chroot-cache)
            # pass this flag to avoid rebuilding chroot again if it's already
            # been built
            USE_CHROOT_CACHE=true
            ;;
        --use-chroot-archive)
            # pass this flag to specify a chroot to use instead of the cache
            USE_CHROOT_CACHE=true
            CHROOT_ARCHIVE="$2"
            shift
            ;;
        --no-cleanup)
            # pass this flag to keep the lxd build environment after build has
            # finished or failed
            CLEANUP=false
            ;;
        --image-target)
            # Specify a class of hooks to build, corresponding to those
            # specified in <project>/hooks.d/extra/series
            # e.g. 'ec2'
            IMAGE_TARGET="$IMAGE_TARGET --image-target $2"
            shift
            ;;
        --repo-snapshot-stamp)
            # Specify a snapshot of the apt package repo for use across
            # parallel builds
            # input to this argument is the output of `date +%s`
            # e.g. '1563222269'
            REPO_SNAPSHOT="--repo-snapshot-stamp $2"
            shift
            ;;
        --snap-cohort-key)
            # specify a snap cohort key to include the same set of snaps across
            # builds (this is occasionally broken by the snap store)
            # As of 20190715 this snap feature is still in beta
            SNAP_COHORT="--cohort-key $2"
            shift
            ;;
        -?*)
            echo "WARNING: Unknown option or argument ignored: $1"
            exit 1
            ;;
        *)
            break
    esac

    shift
done

case `arch` in
    aarch64) ARCH=arm64;;
    x86_64) ARCH=amd64;;
    ppc64le) ARCH=ppc64el;;
    *) echo "Unrecognized arch: $(arch)"; exit 1;;
esac

if [ x$SERIES = "x" ] ; then
    echo "Missing required '--series' argument e.g. '--series xenial'"
    exit 1
fi

DEPENDENCIES=(launchpad-buildd bzr python3-ubuntutools python3-launchpadlib)

for dep in ${DEPENDENCIES[@]} ; do
    if ! [[ $(dpkg -l | grep ^ii | grep $dep) != "" ]] ; then
        echo "Please install the required dependencies for this script:"
        echo "  sudo add-apt-repository -y -u ppa:launchpad/ubuntu/buildd"
        echo "  sudo apt install -y ${DEPENDENCIES[@]}"
        exit 1
    fi
done

if ! [ -f get-ppa-fingerprint ] || ! [ -d live-build ] ; then
    echo 'Cannot find the expected livecd-rootfs tree structure in $PWD.'
    echo 'Are you sure you are running this from a livecd-rootfs branch?'
    exit 1
fi

# Setup code
#*******************
# The real script

set -xe

OLD_FASHIONED_BUILD_CACHE="/tmp/old-fashioned-builder"
UAT_CHECKOUT="$OLD_FASHIONED_BUILD_CACHE/uat"
CHROOT_ARCHIVE_NAME="chroot-ubuntu-$SERIES-${ARCH}.tar.bz"
CHROOT_ARCHIVE="${CHROOT_ARCHIVE:-$OLD_FASHIONED_BUILD_CACHE/$CHROOT_ARCHIVE_NAME}"
OUTPUT_DIRECTORY=~/build.output
LIVEFS_NAME="LOCAL_IMAGES_BUILD"
SERIAL="$(date +%Y%m%d.%H%M)"

mkdir -p $OLD_FASHIONED_BUILD_CACHE

# Get the chroot filesystem from Launchpad if there isn't already one locally
# we could reuse.
if ! [ -f $CHROOT_ARCHIVE ] || [ $USE_CHROOT_CACHE = false ] ; then
    echo "Downloading chroot filesystem from launchpad."
    rm -rf $UAT_CHECKOUT
    git clone https://git.launchpad.net/ubuntu-archive-tools $UAT_CHECKOUT
    $UAT_CHECKOUT/manage-chroot -a ${ARCH} -s $SERIES info
    $UAT_CHECKOUT/manage-chroot -a ${ARCH} -s $SERIES -f $CHROOT_ARCHIVE get
else
    echo "Reusing chroot tarball $CHROOT_ARCHIVE"
fi

# Set the time with NTP, untar the chroot, mount it.
# Use a public NTP server
sudo sed -i 's/ntp\.buildd/0\.pool\.ntp\.org/g' \
  /etc/launchpad-buildd/default

/usr/share/launchpad-buildd/bin/builder-prep
/usr/share/launchpad-buildd/bin/in-target unpack-chroot --backend=lxd \
  --series=$SERIES --arch=${ARCH} $LIVEFS_NAME $CHROOT_ARCHIVE
/usr/share/launchpad-buildd/bin/in-target mount-chroot --backend=lxd \
  --series=$SERIES --arch=${ARCH} $LIVEFS_NAME

# Inject squid proxy config in the LXC if one exists in the host.
SQUID_ADDRESS=$(cat /etc/apt/apt.conf.d/* | sed -n 's/Acquire::http::Proxy "\(.*\)";/\1/p')
SQUID_TMP_CONFIG=/tmp/30squid-proxy
HTTP_PROXY=""
if [ x"$SQUID_ADDRESS" != 'x' ] ; then
    cat > /tmp/30squid-proxy << EOF
Acquire::http::Proxy "$SQUID_ADDRESS";
EOF
    lxc file push $SQUID_TMP_CONFIG lp-$SERIES-${ARCH}/etc/apt/apt.conf.d/30squid-proxy
    lxc config set lp-$SERIES-${ARCH} environment.LB_APT_HTTP_PROXY "$SQUID_ADDRESS"
fi

# Use the same apt mirror as the host
export MIRROR=$(egrep "(archive|ports)" /etc/apt/sources.list|head -1 | \
               cut -d' ' -f2 | cut -d'/' -f3)
export MIRROR_PATH=$(egrep "(archive|ports)" /etc/apt/sources.list|head -1 | \
               cut -d' ' -f2 | cut -d'/' -f4)

proposed_source=""

if [ "$PROPOSED_IN_IMAGE" = "true" ]; then
  proposed_source="deb http://$MIRROR/${MIRROR_PATH} $SERIES-proposed main restricted universe multiverse"
fi

/usr/share/launchpad-buildd/bin/in-target override-sources-list \
  --backend=lxd --series=$SERIES --arch=${ARCH} $LIVEFS_NAME \
    "deb http://$MIRROR/${MIRROR_PATH} $SERIES main restricted universe multiverse" \
    "deb http://$MIRROR/${MIRROR_PATH} $SERIES-security main restricted universe multiverse" \
    "deb http://$MIRROR/${MIRROR_PATH} $SERIES-updates main restricted universe multiverse" \
    "${proposed_source}"
  
/usr/share/launchpad-buildd/bin/in-target update-debian-chroot \
  --backend=lxd --series=$SERIES --arch=${ARCH} $LIVEFS_NAME

# Inject the files from the current tree in the right place in the LXD
# container.
# First install livecd-rootfs and live-build. We will replace the bulk of the livecd-rootfs
# code with our local copy. Without this livecd-rootfs is installed by
# buildlivefs (below) overwriting changes
# If we have specified a local live-build directory then we will replace the
# installed live-build code with our local copy. Without this live-build is
# installed by buildlivefs (below) overwriting changes
lxc exec lp-$SERIES-${ARCH} -- apt-get install --assume-yes livecd-rootfs live-build

# Remove and recreate the livecd-rootfs code we will be replacing with code
# in our current directory
lxc exec lp-$SERIES-${ARCH} -- rm --recursive --force /usr/share/livecd-rootfs
lxc exec lp-$SERIES-${ARCH} -- mkdir --parents /usr/share/livecd-rootfs
# Old LXCs don't have recursive push... so we tar and untar instead.
ls --all -l
tar --create --gzip --verbose --file $OLD_FASHIONED_BUILD_CACHE/livecd-rootfs.tar.gz .
lxc file push $OLD_FASHIONED_BUILD_CACHE/livecd-rootfs.tar.gz lp-$SERIES-${ARCH}/usr/share/livecd-rootfs/
lxc exec lp-$SERIES-${ARCH} -- tar --extract --gzip --verbose --file /usr/share/livecd-rootfs/livecd-rootfs.tar.gz -C /usr/share/livecd-rootfs/
# remove the tarball as it is no longer needed
lxc exec lp-$SERIES-${ARCH} -- rm --verbose /usr/share/livecd-rootfs/livecd-rootfs.tar.gz

# If the $HOME/live-build directory exists, copy it into the LXD container overwriting the installed live-build scripts
if [ -d $HOME/live-build ] ; then
  pushd $HOME/live-build
  # create a tarball of the live-build code
  tar --create --gzip --verbose --file $OLD_FASHIONED_BUILD_CACHE/live-build.tar.gz .
  # create a temporary directory in the container to hold the live-build code
  lxc exec lp-$SERIES-${ARCH} -- mkdir --parents /tmp/live-build
  # push the tarball into the container to the temporary directory
  lxc file push $OLD_FASHIONED_BUILD_CACHE/live-build.tar.gz lp-$SERIES-${ARCH}/tmp/live-build/
  # extract the tarball into the temporary live-build directory
  lxc exec lp-$SERIES-${ARCH} -- tar --extract --gzip --verbose --file /tmp/live-build/live-build.tar.gz -C /tmp/live-build/
  # remove the tarball as it is no longer needed
  lxc exec lp-$SERIES-${ARCH} -- rm --verbose /tmp/live-build/live-build.tar.gz

  # Now copy the live-build code into the installed locations in the container
  # These steps were taken from the install section of the live-build debian/rules file
  #mkdir -p $(DESTDIR)/usr/share/live/build
  lxc exec lp-$SERIES-${ARCH} -- mkdir --verbose --parents /usr/share/live/build
  #cp -r frontends/cgi data functions templates VERSION $(DESTDIR)/usr/share/live/build
  lxc exec lp-$SERIES-${ARCH} -- cp --recursive --verbose /tmp/live-build/frontends/cgi /tmp/live-build/data /tmp/live-build/functions /tmp/live-build/templates /tmp/live-build/VERSION /usr/share/live/build
  #cp -r share/* $(DESTDIR)/usr/share/live/build
  lxc exec lp-$SERIES-${ARCH} -- bash -c "cp --archive --verbose /tmp/live-build/share/* /usr/share/live/build"

  #
  ## Installing executables
  #mkdir -p $(DESTDIR)/usr/bin
  lxc exec lp-$SERIES-${ARCH} -- mkdir --verbose --parents /usr/bin
  #cp -a bin/* $(DESTDIR)/usr/bin
  lxc exec lp-$SERIES-${ARCH} -- bash -c "cp --archive --verbose /tmp/live-build/bin/* /usr/bin"
  #
  #mkdir -p $(DESTDIR)/usr/lib/live
  lxc exec lp-$SERIES-${ARCH} -- mkdir --verbose --parents /usr/lib/live
  #cp -a scripts/* $(DESTDIR)/usr/lib/live
  lxc exec lp-$SERIES-${ARCH} -- bash -c "cp --archive --verbose /tmp/live-build/scripts/* /usr/lib/live"

  lxc exec lp-$SERIES-${ARCH} -- chmod +x /usr/lib/live/build/lb_chroot_early_hooks

  popd
fi

# Use qemu-backports to build xenial images. Otherwise, the images may not boot.
if [ "$SERIES" = "xenial" ]; then
  lxc exec lp-$SERIES-${ARCH} -- apt-get install --assume-yes software-properties-common
  lxc exec lp-$SERIES-${ARCH} -- add-apt-repository --yes ppa:cloud-images-release-managers/qemu-backports
  lxc exec lp-$SERIES-${ARCH} -- apt-get update
  lxc exec lp-$SERIES-${ARCH} -- apt-get install --assume-yes qemu-utils
fi

# Actually build.
time /usr/share/launchpad-buildd/bin/in-target buildlivefs \
  --backend=lxd --series=$SERIES --arch=${ARCH} $LIVEFS_NAME $HTTP_PROXY \
  --project $PROJECT $SUBPROJECT --datestamp $SERIAL --image-format $IMG_FORMAT --debug \
  $EXTRA_PPA \
  $IMAGE_TARGET $REPO_SNAPSHOT $SNAP_COHORT $PROPOSED_BUILD $DEBUG

echo "Copying files out to $OUTPUT_DIRECTORY"
rm -rf $OUTPUT_DIRECTORY
mkdir $OUTPUT_DIRECTORY
lxc file pull lp-$SERIES-${ARCH}/build/binary.log $OUTPUT_DIRECTORY/;
time for FILE in `lxc exec lp-$SERIES-${ARCH} -- find /build -mindepth 1 \
                  \! -type d \! -type l -name 'livecd.*' -print`; do
    echo $FILE
    lxc file pull lp-$SERIES-${ARCH}$FILE $OUTPUT_DIRECTORY/
done

if [ $CLEANUP = false ] ; then
    echo "Exiting without cleanup, --no-cleanup specified"
    exit 0
fi

# Cleanup the builder LXD.
/usr/share/launchpad-buildd/bin/in-target scan-for-processes \
  --backend=lxd --series=$SERIES --arch=${ARCH} $LIVEFS_NAME

/usr/share/launchpad-buildd/bin/in-target umount-chroot \
  --backend=lxd --series=$SERIES --arch=${ARCH} $LIVEFS_NAME

/usr/share/launchpad-buildd/bin/in-target remove-build \
  --backend=lxd --series=$SERIES --arch=${ARCH} $LIVEFS_NAME