Build all Erlang versions and upload to PackageCloud (#6)

Use a multi step Github Action workflow, first identify which versions from https://github.com/erlang/otp/releases that are missing from https://packagecloud.io/cloudamqp/erlang. Then build each version/distribution/platform combination that is missing in a matrix.

.github/workflows/build-all-and-upload.yml

@@ -0,0 +1,51 @@
+name: Build all Erlang versions and upload to PackageCloud
+
+on:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '30 8 * * 1-5' # 08:30 mon-fri
+  push:
+    branches:
+      - main
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+jobs:
+  missing-versions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby
+      - name: Check missing versions
+        id: missing-versions
+        run: echo "matrix=$(bin/missing-versions)" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+    outputs:
+      matrix: ${{ steps.missing-versions.outputs.matrix }}
+
+  build-and-upload:
+    needs: missing-versions
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 10
+      matrix:
+        include: ${{ fromJson(needs.missing-versions.outputs.matrix) }}
+      fail-fast: false
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby
+      - uses: depot/use-action@v1
+      - name: Build Erlang version and upload to PackageCloud
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+        run : bin/build-and-upload ${{ matrix.version }} ${{ matrix.image }} ${{ matrix.platform }}

Dockerfile

@@ -1,37 +1,28 @@
 ARG image=ubuntu:jammy
-FROM --platform=$BUILDPLATFORM ${image} AS builder
-ARG BUILDARCH
-ARG TARGETARCH
+FROM ${image} AS builder
 ARG DEBIAN_FRONTEND=noninteractive
-RUN dpkg --add-architecture $TARGETARCH && \
-    . /etc/os-release && \
-    if [ "$ID" = ubuntu ]; then \
-        sed -i "s/^deb /deb [arch=$BUILDARCH] /" /etc/apt/sources.list; \
-        echo "deb [arch=arm64] http://ports.ubuntu.com/ $VERSION_CODENAME main universe" >> /etc/apt/sources.list; \
-        echo "deb [arch=arm64] http://ports.ubuntu.com/ $VERSION_CODENAME-updates main universe" >> /etc/apt/sources.list; \
-    fi && \
-    apt-get update
+RUN apt-get update
 
 RUN apt-get install -y curl build-essential pkg-config ruby binutils autoconf libwxbase3.0-dev \
-                       libssl-dev:$TARGETARCH libtinfo-dev:$TARGETARCH zlib1g-dev:$TARGETARCH libsnmp-dev:$TARGETARCH && \
+                       libssl-dev libtinfo-dev zlib1g-dev libsnmp-dev && \
     (ruby -e "exit RUBY_VERSION.to_f > 2.5" || gem install --no-document public_suffix -v 4.0.7) && \
+    (ruby -e "exit RUBY_VERSION.to_f >= 3.0" || gem install --no-document dotenv -v 2.8.1 ) && \
     gem install --no-document fpm
-RUN if [ "$TARGETARCH" = arm64 ]; then apt-get install -y crossbuild-essential-arm64 binutils-aarch64-linux-gnu; fi
 
 WORKDIR /tmp/openssl
 ARG erlang_version=24.0
 # Erlang before 24.2 didn't support libssl3, so statically compile 1.1.1 if no available from the OS
 RUN libssl_version=$(dpkg-query --showformat='${Version}' --show libssl-dev); \
     if (dpkg --compare-versions "$erlang_version" ge 20.0 && dpkg --compare-versions "$erlang_version" lt 24.2 && dpkg --compare-versions "$libssl_version" ge 3.0.0); then \
         curl https://www.openssl.org/source/openssl-1.1.1t.tar.gz | tar zx --strip-components=1 && \
-        ./Configure no-shared $([ "$TARGETARCH" = arm64 ] && echo "linux-aarch64 --cross-compile-prefix=aarch64-linux-gnu-" || echo "linux-x86_64") && \
+        ./config no-shared && \
         make -j$(nproc) && make install_sw; \
     fi
 
 # Erlang before 20.0 didn't support libssl1.1, so statically compile 1.0.2
 RUN if (dpkg --compare-versions "$erlang_version" lt 20.0); then \
         curl https://www.openssl.org/source/old/1.0.2/openssl-1.0.2u.tar.gz | tar zx --strip-components=1 && \
-        ./Configure --prefix=/usr/local --openssldir=/usr/local/ssl no-shared $([ "$TARGETARCH" = arm64 ] && echo "linux-aarch64 --cross-compile-prefix=aarch64-linux-gnu-" || echo "linux-x86_64") "-fPIC" && \
+        ./config --prefix=/usr/local --openssldir=/usr/local/ssl no-shared -fPIC && \
         make -j$(nproc) && make install_sw; \
     fi
 
@@ -43,20 +34,13 @@ RUN if (grep -q jammy /etc/os-release && dpkg --compare-versions "$erlang_versio
         apt-get install -y gcc-9 autoconf2.69 && \
         ln -sf /usr/bin/gcc-9 /usr/bin/gcc && \
         ln -sf /usr/bin/autoconf2.69 /usr/bin/autoconf; \
-        if [ "$TARGETARCH" = arm64 ] && [ "$BUILDARCH" != arm64 ]; then \
-            apt-get install -y gcc-9-aarch64-linux-gnu && \
-            ln -sf /usr/bin/aarch64-linux-gnu-gcc-9 /usr/bin/aarch64-linux-gnu-gcc; \
-        fi \
     fi
 
 ARG CFLAGS="-g -O2 -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security"
 ARG CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
 ARG LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro"
 ARG ERLC_USE_SERVER=false
 RUN ./otp_build autoconf
-RUN if [ "$TARGETARCH" = arm64 ]; then \
-        ./configure --enable-bootstrap-only && make -j$(nproc); \
-    fi
 RUN libssl_version=$(dpkg-query --showformat='${Version}' --show libssl-dev); \
     STATIC_OPENSSL=$(dpkg --compare-versions "$erlang_version" lt 20 || (dpkg --compare-versions "$erlang_version" lt 24.2 && dpkg --compare-versions "$libssl_version" ge 3) && echo y); \
     ./configure erl_xcomp_sysroot=/ \
@@ -77,21 +61,18 @@ RUN libssl_version=$(dpkg-query --showformat='${Version}' --show libssl-dev); \
                 --without-eunit \
                 --with-ssl-rpath=no \
                 --with-ssl \
-                $([ "$TARGETARCH" = arm64 ] && echo "--host=aarch64-linux-gnu --build=$BUILDARCH-linux-gnu") \
                 $([ "$STATIC_OPENSSL" = y ] && echo "--with-ssl=/usr/local --disable-dynamic-ssl-lib" || echo --enable-dynamic-ssl-lib) && \
     make -j$(nproc) && \
     make install DESTDIR=/tmp/install && \
     find /tmp/install -type d -name examples | xargs rm -r && \
-    find /tmp/install -type f -executable -exec $([ "$TARGETARCH" = arm64 ] && echo aarch64-linux-gnu-)strip {} \;;
+    find /tmp/install -type f -executable -exec strip {} \;;
 # when cross compiling the target version of strip is required
 
 ARG erlang_iteration=1
-RUN readelf=$([ "$TARGETARCH" = arm64 ] && echo aarch64-linux-gnu-)readelf; \
-    fpm -s dir -t deb \
+RUN fpm -s dir -t deb \
     --chdir /tmp/install \
     --name esl-erlang \
     --version $erlang_version \
-    --architecture $TARGETARCH \
     --epoch 1 \
     --iteration $erlang_iteration \
     --maintainer "CloudAMQP <[email protected]>" \
@@ -100,7 +81,7 @@ RUN readelf=$([ "$TARGETARCH" = arm64 ] && echo aarch64-linux-gnu-)readelf; \
     --url "https://erlang.org" \
     --license "Apache 2.0" \
     --depends "procps" \
-    --depends "$($readelf -d $(find /tmp/install/usr -name beam.smp) | awk '/NEEDED/{gsub(/[\[\]]/, "");print $5}' | xargs dpkg -S | cut -d: -f1 | sort -u | paste -sd,)" \
+    --depends "$(readelf -d $(find /tmp/install/usr -name beam.smp) | awk '/NEEDED/{gsub(/[\[\]]/, "");print $5}' | xargs dpkg -S | cut -d: -f1 | sort -u | paste -sd,)" \
     --conflicts "erlang-asn1,erlang-base,erlang-base-hipe,erlang-common-test,erlang-corba,erlang-crypto,erlang-debugger,erlang-dev,erlang-dialyzer,erlang-diameter,erlang-doc,erlang-edoc,erlang-eldap,erlang-erl-docgen,erlang-et,erlang-eunit,erlang-examples,erlang-ftp,erlang-ic,erlang-ic-java,erlang-inets,erlang-inviso,erlang-jinterface,erlang-manpages,erlang-megaco,erlang-mnesia,erlang-mode,erlang-nox,erlang-observer,erlang-odbc,erlang-os-mon,erlang-parsetools,erlang-percept,erlang-public-key,erlang-reltool,erlang-runtime-tools,erlang-snmp,erlang-src,erlang-ssh,erlang-ssl,erlang-syntax-tools,erlang-tftp,erlang-tools,erlang-webtool,erlang-wx,erlang-xmerl"
 
 #RUN apt-get install -y lintian

bin/build-and-upload

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+require_relative "../lib/packagecloud"
+
+if ARGV.size != 3
+  abort "#{File.basename $PROGRAM_NAME} build-and-upload <version> <image> <platform>"
+end
+
+version, image, platform = ARGV.shift(3)
+system("depot", "build",
+       "--platform", "linux/#{platform}",
+       "--build-arg", "erlang_version=#{version}",
+       "--build-arg", "image=#{image.sub('/', ':')}",
+       "--output", ".",
+       ".", exception: true)
+
+packagecloud = Packagecloud.new
+File.open("esl-erlang_#{version}-1_#{platform}.deb") do |file|
+  packagecloud.upload(image.sub(":", "/"), file)
+  File.unlink(file)
+end

bin/missing-versions

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require_relative "../lib/github"
+require_relative "../lib/packagecloud"
+
+DISTS = %w[ubuntu/jammy ubuntu/focal].freeze
+PLATFORMS = %w[amd64 arm64].freeze
+
+packagecloud = Packagecloud.new
+github = Github.new
+
+missing = []
+github.releases do |r|
+  next if r["prerelease"]
+  next if r["draft"]
+  next if r["tag_name"].include? "-rc"
+  version = r["tag_name"].sub("OTP-", "")
+  DISTS.each do |dist|
+    PLATFORMS.each do |platform|
+      filename = "esl-erlang_#{version}-1_#{platform}.deb"
+      next if packagecloud.exists? dist, filename
+      image = dist.sub("/", ":")
+      missing << { version:, image:, platform: }
+    end
+  end
+end
+
+# Output for Github Action
+JSON.dump(missing.take(256), $stdout)
+
+# Github Actions support maximum 256 jobs
+warn "Result truncated to 256, actual missing versions: #{to_build.size}" if to_build.size > 256

depot.json

@@ -0,0 +1 @@
+{"id":"srqn8st7l7"}

lib/github.rb

@@ -0,0 +1,20 @@
+require "net/http"
+require "json"
+
+class Github
+  def initialize(token = ENV.fetch("GITHUB_TOKEN"))
+    @auth = { Authorization: "Bearer #{token}" }
+  end
+
+  def releases(&blk)
+    Net::HTTP.start("api.github.com", use_ssl: true) do |api|
+      1.upto(10).each do |page|
+        resp = api.get("/repos/erlang/otp/releases?per_page=100&&page=#{page}", @auth)
+        raise "Unexpected response: #{resp} #{resp.body}" unless Net::HTTPOK === resp
+        releases = JSON.parse(resp.body)
+        break if releases.empty?
+        releases.each(&blk)
+      end
+    end
+  end
+end