Docker: Build and use musl-based binaries in alpine images to resolve glibc incompatibility issues (grafana#19798)

* build: Install musl cross compilers as part of build Docker image
* build: Build also musl packages in scripts/build/build.sh
* scripts/build/build-all.sh: Build musl Linux targets
* build: Upgrade build-container to 1.2.11
* build.go: De-duplicate code
* build: Base Docker images on musl binaries

Author: aknuds1

.circleci/config.yml

@@ -195,7 +195,7 @@ jobs:
 
   build-all:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -239,7 +239,7 @@ jobs:
 
   build:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -268,7 +268,7 @@ jobs:
 
   build-fast-backend:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -285,7 +285,7 @@ jobs:
 
   build-fast-frontend:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -309,7 +309,7 @@ jobs:
 
   build-fast-package:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -336,7 +336,7 @@ jobs:
 
   build-fast-save:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -425,7 +425,7 @@ jobs:
 
   build-enterprise:
     docker:
-     - image: grafana/build-container:1.2.10
+     - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
@@ -460,7 +460,7 @@ jobs:
 
   build-all-enterprise:
     docker:
-    - image: grafana/build-container:1.2.10
+    - image: grafana/build-container:1.2.11
     working_directory: /go/src/github.com/grafana/grafana
     steps:
     - checkout

Gruntfile.js

@@ -10,6 +10,7 @@ module.exports = function (grunt) {
     tempDir: 'tmp',
     platform: process.platform.replace('win32', 'windows'),
     enterprise: false,
+    libc: null,
   };
 
   if (grunt.option('platform')) {
@@ -30,6 +31,10 @@ module.exports = function (grunt) {
     }
   }
 
+  if (grunt.option('libc')) {
+    config.libc = grunt.option('libc');
+  }
+
   config.phjs = grunt.option('phjsToRelease');
   config.pkg.version = grunt.option('pkgVer') || config.pkg.version;
 
@@ -42,7 +47,7 @@ module.exports = function (grunt) {
   grunt.loadTasks('./scripts/grunt');
 
   // Utility function to load plugin settings into config
-  function loadConfig(config,path) {
+  function loadConfig(config, path) {
     require('glob').sync('*', {cwd: path}).forEach(function(option) {
       var key = option.replace(/\.js$/,'');
       // If key already exists, extend it. It is your responsibility to avoid naming collisions

build.go

@@ -33,6 +33,7 @@ var (
 	goos    string
 	gocc    string
 	cgo     bool
+	libc    string
 	pkgArch string
 	version string = "v1"
 	// deb & rpm does not support semver so have to handle their version a little differently
@@ -65,6 +66,7 @@ func main() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
 	flag.StringVar(&gocc, "cc", "", "CC")
+	flag.StringVar(&libc, "libc", "", "LIBC")
 	flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo")
 	flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH")
 	flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary")
@@ -106,18 +108,14 @@ func main() {
 		case "setup":
 			setup()
 
-		case "build-srv":
+		case "build-srv", "build-server":
 			clean()
 			build("grafana-server", "./pkg/cmd/grafana-server", []string{})
 
 		case "build-cli":
 			clean()
 			build("grafana-cli", "./pkg/cmd/grafana-cli", []string{})
 
-		case "build-server":
-			clean()
-			build("grafana-server", "./pkg/cmd/grafana-server", []string{})
-
 		case "build":
 			//clean()
 			for _, binary := range binaries {
@@ -176,12 +174,15 @@ func makeLatestDistCopies() {
 	}
 
 	latestMapping := map[string]string{
-		"_amd64.deb":          "dist/grafana_latest_amd64.deb",
-		".x86_64.rpm":         "dist/grafana-latest-1.x86_64.rpm",
-		".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
-		".linux-armv7.tar.gz": "dist/grafana-latest.linux-armv7.tar.gz",
-		".linux-armv6.tar.gz": "dist/grafana-latest.linux-armv6.tar.gz",
-		".linux-arm64.tar.gz": "dist/grafana-latest.linux-arm64.tar.gz",
+		"_amd64.deb":               "dist/grafana_latest_amd64.deb",
+		".x86_64.rpm":              "dist/grafana-latest-1.x86_64.rpm",
+		".linux-amd64.tar.gz":      "dist/grafana-latest.linux-x64.tar.gz",
+		".linux-amd64-musl.tar.gz": "dist/grafana-latest.linux-x64-musl.tar.gz",
+		".linux-armv7.tar.gz":      "dist/grafana-latest.linux-armv7.tar.gz",
+		".linux-armv7-musl.tar.gz": "dist/grafana-latest.linux-armv7-musl.tar.gz",
+		".linux-armv6.tar.gz":      "dist/grafana-latest.linux-armv6.tar.gz",
+		".linux-arm64.tar.gz":      "dist/grafana-latest.linux-arm64.tar.gz",
+		".linux-arm64-musl.tar.gz": "dist/grafana-latest.linux-arm64-musl.tar.gz",
 	}
 
 	for _, file := range files {
@@ -455,6 +456,9 @@ func gruntBuildArg(task string) []string {
 	if pkgArch != "" {
 		args = append(args, fmt.Sprintf("--arch=%v", pkgArch))
 	}
+	if libc != "" {
+		args = append(args, fmt.Sprintf("--libc=%s", libc))
+	}
 	if phjsToRelease != "" {
 		args = append(args, fmt.Sprintf("--phjsToRelease=%v", phjsToRelease))
 	}
@@ -481,9 +485,13 @@ func test(pkg string) {
 }
 
 func build(binaryName, pkg string, tags []string) {
-	binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName)
+	libcPart := ""
+	if libc != "" {
+		libcPart = fmt.Sprintf("-%s", libc)
+	}
+	binary := fmt.Sprintf("./bin/%s-%s%s/%s", goos, goarch, libcPart, binaryName)
 	if isDev {
-		//don't include os and arch in output path in dev environment
+		//don't include os/arch/libc in output path in dev environment
 		binary = fmt.Sprintf("./bin/%s", binaryName)
 	}
 
@@ -511,7 +519,11 @@ func build(binaryName, pkg string, tags []string) {
 	if !isDev {
 		setBuildEnv()
 		runPrint("go", "version")
-		fmt.Printf("Targeting %s/%s\n", goos, goarch)
+		libcPart := ""
+		if libc != "" {
+			libcPart = fmt.Sprintf("/%s", libc)
+		}
+		fmt.Printf("Targeting %s/%s%s\n", goos, goarch, libcPart)
 	}
 
 	runPrint("go", args...)

packaging/docker/Dockerfile

@@ -1,7 +1,7 @@
 ARG BASE_IMAGE=alpine:3.10
 FROM ${BASE_IMAGE}
 
-ARG GRAFANA_TGZ="grafana-latest.linux-x64.tar.gz"
+ARG GRAFANA_TGZ="grafana-latest.linux-x64-musl.tar.gz"
 
 COPY ${GRAFANA_TGZ} /tmp/grafana.tar.gz
 

packaging/docker/build.sh

@@ -59,10 +59,10 @@ docker_tag_all () {
 	fi
 }
 
-docker_build "alpine:3.10" "grafana-latest.linux-x64.tar.gz" "${_docker_repo}:${_grafana_version}"
+docker_build "alpine:3.10" "grafana-latest.linux-x64-musl.tar.gz" "${_docker_repo}:${_grafana_version}"
 if [ $BUILD_FAST = "0" ]; then
-	docker_build "arm32v7/alpine:3.10" "grafana-latest.linux-armv7.tar.gz" "${_docker_repo}-arm32v7-linux:${_grafana_version}"
-	docker_build "arm64v8/alpine:3.10" "grafana-latest.linux-arm64.tar.gz" "${_docker_repo}-arm64v8-linux:${_grafana_version}"
+	docker_build "arm32v7/alpine:3.10" "grafana-latest.linux-armv7-musl.tar.gz" "${_docker_repo}-arm32v7-linux:${_grafana_version}"
+	docker_build "arm64v8/alpine:3.10" "grafana-latest.linux-arm64-musl.tar.gz" "${_docker_repo}-arm64v8-linux:${_grafana_version}"
 fi
 # Tag as 'latest' for official release; otherwise tag as grafana/grafana:master
 if echo "$_grafana_tag" | grep -q "^v"; then

scripts/build/build-all.sh

@@ -13,10 +13,13 @@ EXTRA_OPTS="$@"
 
 CCARMV6=/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc
 CCARMV7=arm-linux-gnueabihf-gcc
+CCARMV7_MUSL=/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc
 CCARM64=aarch64-linux-gnu-gcc
+CCARM64_MUSL=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc
 CCOSX64=/tmp/osxcross/target/bin/o64-clang
 CCWIN64=x86_64-w64-mingw32-gcc
 CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
+CCX64_MUSL=/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc
 
 cd /go/src/github.com/grafana/grafana
 echo "current dir: $(pwd)"
@@ -37,13 +40,17 @@ if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
   go run build.go -goarch armv6 -cc ${CCARMV6} ${OPT} build
   go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
   go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
+  go run build.go -goarch armv7 -libc musl -cc ${CCARMV7_MUSL} ${OPT} build
+  go run build.go -goarch arm64 -libc musl -cc ${CCARM64_MUSL} ${OPT} build
   go run build.go -goos darwin -cc ${CCOSX64} ${OPT} build
 fi
 
 go run build.go -goos windows -cc ${CCWIN64} ${OPT} build
 
 # Do not remove CC from the linux build, its there for compatibility with Centos6
-CC=${CCX64} go run build.go ${OPT} build
+go run build.go -cc ${CCX64} ${OPT} build
+
+go run build.go -cc ${CCX64_MUSL} -libc musl ${OPT} build
 
 yarn install --pure-lockfile --no-progress
 
@@ -68,6 +75,7 @@ source /etc/profile.d/rvm.sh
 
 echo "Packaging"
 go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
+go run build.go -goos linux -pkg-arch amd64 -libc musl ${OPT} -skipRpm -skipDeb package-only
 #removing amd64 phantomjs bin for armv7/arm64 packages
 rm tools/phantomjs/phantomjs
 
@@ -76,6 +84,8 @@ if echo "$EXTRA_OPTS" | grep -vq enterprise ; then
   go run build.go -goos linux -pkg-arch armv6 ${OPT} -skipRpm package-only
   go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
   go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
+  go run build.go -goos linux -pkg-arch armv7 -libc musl ${OPT} -skipRpm -skipDeb package-only
+  go run build.go -goos linux -pkg-arch arm64 -libc musl ${OPT} -skipRpm -skipDeb package-only
 
   if [ -d '/tmp/phantomjs/darwin' ]; then
     cp /tmp/phantomjs/darwin/phantomjs tools/phantomjs/phantomjs

scripts/build/build.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+
+# shellcheck disable=SC2086
+
 #
 #   This script is executed from within the container.
 #
@@ -7,8 +10,11 @@ set -e
 ##########
 CCARMV6=/opt/rpi-tools/arm-bcm2708/arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc
 CCARMV7=arm-linux-gnueabihf-gcc
+CCARMV7_MUSL=/tmp/arm-linux-musleabihf-cross/bin/arm-linux-musleabihf-gcc
 CCARM64=aarch64-linux-gnu-gcc
+CCARM64_MUSL=/tmp/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc
 CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
+CCX64_MUSL=/tmp/x86_64-linux-musl-cross/bin/x86_64-linux-musl-gcc
 
 BUILD_FAST=0
 BUILD_BACKEND=1
@@ -69,17 +75,20 @@ function build_backend_linux_amd64() {
   if [ ! -d "dist" ]; then
     mkdir dist
   fi
-  CC=${CCX64} go run build.go "${OPT}" build
+  CC=${CCX64} go run build.go ${OPT} build
+  CC=${CCX64_MUSL} go run build.go -libc musl ${OPT} build
 }
 
 function build_backend() {
   if [ ! -d "dist" ]; then
     mkdir dist
   fi
 
-  go run build.go -goarch armv6 -cc ${CCARMV6} "${OPT}" build
-  go run build.go -goarch armv7 -cc ${CCARMV7} "${OPT}" build
-  go run build.go -goarch arm64 -cc ${CCARM64} "${OPT}" build
+  go run build.go -goarch armv6 -cc ${CCARMV6} ${OPT} build
+  go run build.go -goarch armv7 -cc ${CCARMV7} ${OPT} build
+  go run build.go -goarch arm64 -cc ${CCARM64} ${OPT} build
+  go run build.go -goarch armv7 -libc musl -cc ${CCARMV7_MUSL} ${OPT} build
+  go run build.go -goarch arm64 -libc musl -cc ${CCARM64_MUSL} ${OPT} build
   build_backend_linux_amd64
 }
 
@@ -89,22 +98,25 @@ function build_frontend() {
   fi
   yarn install --pure-lockfile --no-progress
   echo "Building frontend"
-  go run build.go "${OPT}" build-frontend
+  go run build.go ${OPT} build-frontend
   echo "FRONTEND: finished"
 }
 
 function package_linux_amd64() {
   echo "Packaging Linux AMD64"
-  go run build.go -goos linux -pkg-arch amd64 "${OPT}" package-only
+  go run build.go -goos linux -pkg-arch amd64 ${OPT} package-only
+  go run build.go -goos linux -pkg-arch amd64 ${OPT} -libc musl -skipRpm -skipDeb package-only
   go run build.go latest
   echo "PACKAGE LINUX AMD64: finished"
 }
 
 function package_all() {
   echo "Packaging ALL"
-  go run build.go -goos linux -pkg-arch armv6 "${OPT}" -skipRpm package-only
-  go run build.go -goos linux -pkg-arch armv7 "${OPT}" package-only
-  go run build.go -goos linux -pkg-arch arm64 "${OPT}" package-only
+  go run build.go -goos linux -pkg-arch armv6 ${OPT} -skipRpm package-only
+  go run build.go -goos linux -pkg-arch armv7 ${OPT} package-only
+  go run build.go -goos linux -pkg-arch arm64 ${OPT} package-only
+  go run build.go -goos linux -pkg-arch armv7 -libc musl -skipRpm -skipDeb ${OPT} package-only
+  go run build.go -goos linux -pkg-arch arm64 -libc musl -skipRpm -skipDeb ${OPT} package-only
   package_linux_amd64
   echo "PACKAGE ALL: finished"
 }
@@ -115,7 +127,7 @@ function package_setup() {
     rm -rf dist
   fi
   mkdir dist
-  go run build.go -gen-version "${OPT}" > dist/grafana.version
+  go run build.go -gen-version ${OPT} > dist/grafana.version
   # Load ruby, needed for packing with fpm
   # shellcheck disable=SC1091
   source /etc/profile.d/rvm.sh

scripts/build/ci-build/Dockerfile

@@ -74,7 +74,10 @@ FROM ubuntu:14.04
 ENV GOVERSION=1.13.1 \
     PATH=/usr/local/go/bin:$PATH \
     GOPATH=/go \
-    NODEVERSION=10.14.2
+    NODEVERSION=10.14.2 \
+    CHKSUM_ARMV7_MUSL=3043bd16a0287b02f3f1045ec5f7fa42bd47bd8329183fc52b179dbb80a56d5f \
+    CHKSUM_ARMV8_MUSL=1a935561ebe3155bfd02b1b25cd7533c81906dd47b1de1fa43b74209130b941c \
+    CHKSUM_AMD64_MUSL=0fada482750176671d24c31b944fae7c3bf0a826011b3decc12d47b21c3cb5c2
 
 COPY --from=toolchain /tmp/x86_64-centos6-linux-gnu.tar.xz /tmp/
 COPY --from=toolchain /tmp/osxcross.tar.xz /tmp/
@@ -108,6 +111,21 @@ RUN apt-get update   && \
       | tar -xz -C /usr/local && \
     git clone https://github.com/raspberrypi/tools.git /opt/rpi-tools --depth=1
 
+# Install musl cross compilers
+RUN cd /tmp && \
+curl -O https://musl.cc/arm-linux-musleabihf-cross.tgz && \
+[ "$(sha256sum arm-linux-musleabihf-cross.tgz|cut -f1 -d ' ')" = "$CHKSUM_ARMV7_MUSL" ] && \
+tar xf arm-linux-musleabihf-cross.tgz && \
+rm arm-linux-musleabihf-cross.tgz && \
+curl -O https://musl.cc/aarch64-linux-musl-cross.tgz && \
+[ "$(sha256sum aarch64-linux-musl-cross.tgz|cut -f1 -d ' ')" = "$CHKSUM_ARMV8_MUSL" ] && \
+tar xf aarch64-linux-musl-cross.tgz && \
+rm aarch64-linux-musl-cross.tgz && \
+curl -O https://musl.cc/x86_64-linux-musl-cross.tgz && \
+[ "$(sha256sum x86_64-linux-musl-cross.tgz|cut -f1 -d ' ')" = "$CHKSUM_AMD64_MUSL" ] && \
+tar xf x86_64-linux-musl-cross.tgz && \
+rm x86_64-linux-musl-cross.tgz
+
 RUN apt-get install -y                           \
         gcc libc-dev make && \
     gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \

scripts/build/ci-build/build-deploy.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eo pipefail
 
-_version="1.2.10"
+_version="1.2.11"
 _tag="grafana/build-container:${_version}"
 
 _dpath=$(dirname "${BASH_SOURCE[0]}")