diff --git a/.github/workflows/build-all-and-upload.yml b/.github/workflows/build-all-and-upload.yml new file mode 100644 index 0000000..0896ba1 --- /dev/null +++ b/.github/workflows/build-all-and-upload.yml @@ -0,0 +1,51 @@ +name: Build all Erlang versions and upload to PackageCloud + +on: + pull_request: + workflow_dispatch: + schedule: + - cron: '30 8 * * 1-5' # 08:30 mon-fri + push: + branches: + - main +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true +jobs: + missing-versions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ruby + - name: Check missing versions + id: missing-versions + run: echo "matrix=$(bin/missing-versions)" >> "$GITHUB_OUTPUT" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + outputs: + matrix: ${{ steps.missing-versions.outputs.matrix }} + + build-and-upload: + needs: missing-versions + runs-on: ubuntu-latest + strategy: + max-parallel: 10 + matrix: + include: ${{ fromJson(needs.missing-versions.outputs.matrix) }} + fail-fast: false + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ruby + - uses: depot/use-action@v1 + - name: Build Erlang version and upload to PackageCloud + env: + PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + run : bin/build-and-upload ${{ matrix.version }} ${{ matrix.image }} ${{ matrix.platform }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 7b2bd1c..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: CI - -on: - pull_request: - push: - branches: - - main - paths-ignore: - - README.md - -jobs: - debian: - name: Debian - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: - - "ubuntu:jammy" - - "ubuntu:focal" - - "ubuntu:bionic" - erlang_version: - - "26.1.2" - - "26.0" - - "25.3.2.7" - - "25.3" - - "25.2.3" - - "25.2.2" - - "25.2.1" - - "25.1.1" - - "25.1" - - "25.0.4" - - "25.0.3" - - "25.0.1" - - "25.0" - - "24.3.4.8" - - "24.3.4.7" - - "24.3.4.6" - - "24.3.4.5" - - "24.3.4.3" - - "24.3.4.2" - - "24.3.4.1" - - "24.3.4" - - "24.2.2" - - "24.2" - - "24.1.7" - - "24.0.5" - - "24.0.4" - - "24.0.2" - - "24.0.1" - - "24.0" - - "23.3.1" - - "23.2.3" - - "23.2.1" - - "23.1" - - "22.3.4.9" - - "22.3.4.1" - - "22.3.4" - - "22.3.2" - - "22.2.4" - - "22.2.3" - - "22.0.7" - - "22.0.1" - - "21.3.8.6" - - "21.3.8.17" - - "21.3.8.16" - - "21.3.8.15" - - "21.2.3" - - "21.2" - - "21.1" - - "21.0" - - "20.3.8.26" - - "20.3.8.22" - - "20.3" - - "20.1" - - "20.0" - - "19.3.6.13" - - "19.3" - - "19.2" - - "19.1" - - "18.3" - - "18.2" - - "18.1" - - "18.0" - exclude: - # compilation fails on bionic, let's skip it for now - - image: "ubuntu:bionic" - erlang_version: "26.1.2" - - image: "ubuntu:bionic" - erlang_version: "26.0" - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Export packages - uses: docker/build-push-action@v3 - with: - cache-from: type=gha,scope=${{ github.workflow }}-${{ matrix.image }}-${{ matrix.erlang_version }} - cache-to: type=gha,mode=max,scope=${{ github.workflow }}-${{ matrix.image }}-${{ matrix.erlang_version }} - platforms: linux/amd64,linux/arm64 - build-args: | - erlang_version=${{ matrix.erlang_version }} - image=${{ matrix.image }} - outputs: pkgs - - - name: Generate artifact name - run: | - echo "IMAGE=$(echo ${{ matrix.image }} | tr : -)" >> $GITHUB_ENV - - name: Upload artifacts - uses: actions/upload-artifact@v3 - with: - name: erlang-${{ matrix.erlang_version }}-${{ env.IMAGE }}-deb - path: pkgs - - name: Upload to Packagecloud - run: | - set -euxo pipefail - ID=$(echo "${{ matrix.image }}" | cut -f1 -d:) - VERSION_CODENAME=$(echo "${{ matrix.image }}" | cut -f2 -d:) - cat > $HOME/.curlrc << EOF - -u "${{ secrets.packagecloud_token }}:" - --no-progress-meter - --fail - EOF - curl -O https://packagecloud.io/api/v1/distributions.json - DIST_ID=$(jq ".deb[] | select(.index_name == \"$ID\").versions[] | select(.index_name == \"$VERSION_CODENAME\").id" distributions.json) - find pkgs -name "*.deb" | xargs -I@ basename @ | xargs -I@ curl -XDELETE "https://packagecloud.io/api/v1/repos/cloudamqp/erlang/$ID/$VERSION_CODENAME/@" || true - find pkgs -name "*.deb" | xargs -I{} curl -XPOST -F "package[distro_version_id]=$DIST_ID" -F "package[package_file]=@{}" https://packagecloud.io/api/v1/repos/cloudamqp/erlang/packages.json - if: ${{ github.ref == 'refs/heads/main' }} diff --git a/Dockerfile b/Dockerfile index 39698e6..b79fda1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,13 @@ ARG image=ubuntu:jammy -FROM --platform=$BUILDPLATFORM ${image} AS builder -ARG BUILDARCH -ARG TARGETARCH +FROM ${image} AS builder ARG DEBIAN_FRONTEND=noninteractive -RUN dpkg --add-architecture $TARGETARCH && \ - . /etc/os-release && \ - if [ "$ID" = ubuntu ]; then \ - sed -i "s/^deb /deb [arch=$BUILDARCH] /" /etc/apt/sources.list; \ - echo "deb [arch=arm64] http://ports.ubuntu.com/ $VERSION_CODENAME main universe" >> /etc/apt/sources.list; \ - echo "deb [arch=arm64] http://ports.ubuntu.com/ $VERSION_CODENAME-updates main universe" >> /etc/apt/sources.list; \ - fi && \ - apt-get update +RUN apt-get update RUN apt-get install -y curl build-essential pkg-config ruby binutils autoconf libwxbase3.0-dev \ - libssl-dev:$TARGETARCH libtinfo-dev:$TARGETARCH zlib1g-dev:$TARGETARCH libsnmp-dev:$TARGETARCH && \ + libssl-dev libtinfo-dev zlib1g-dev libsnmp-dev && \ (ruby -e "exit RUBY_VERSION.to_f > 2.5" || gem install --no-document public_suffix -v 4.0.7) && \ + (ruby -e "exit RUBY_VERSION.to_f >= 3.0" || gem install --no-document dotenv -v 2.8.1 ) && \ gem install --no-document fpm -RUN if [ "$TARGETARCH" = arm64 ]; then apt-get install -y crossbuild-essential-arm64 binutils-aarch64-linux-gnu; fi WORKDIR /tmp/openssl ARG erlang_version=24.0 @@ -24,14 +15,14 @@ ARG erlang_version=24.0 RUN libssl_version=$(dpkg-query --showformat='${Version}' --show libssl-dev); \ if (dpkg --compare-versions "$erlang_version" ge 20.0 && dpkg --compare-versions "$erlang_version" lt 24.2 && dpkg --compare-versions "$libssl_version" ge 3.0.0); then \ curl https://www.openssl.org/source/openssl-1.1.1t.tar.gz | tar zx --strip-components=1 && \ - ./Configure no-shared $([ "$TARGETARCH" = arm64 ] && echo "linux-aarch64 --cross-compile-prefix=aarch64-linux-gnu-" || echo "linux-x86_64") && \ + ./config no-shared && \ make -j$(nproc) && make install_sw; \ fi # Erlang before 20.0 didn't support libssl1.1, so statically compile 1.0.2 RUN if (dpkg --compare-versions "$erlang_version" lt 20.0); then \ curl https://www.openssl.org/source/old/1.0.2/openssl-1.0.2u.tar.gz | tar zx --strip-components=1 && \ - ./Configure --prefix=/usr/local --openssldir=/usr/local/ssl no-shared $([ "$TARGETARCH" = arm64 ] && echo "linux-aarch64 --cross-compile-prefix=aarch64-linux-gnu-" || echo "linux-x86_64") "-fPIC" && \ + ./config --prefix=/usr/local --openssldir=/usr/local/ssl no-shared -fPIC && \ make -j$(nproc) && make install_sw; \ fi @@ -43,10 +34,6 @@ RUN if (grep -q jammy /etc/os-release && dpkg --compare-versions "$erlang_versio apt-get install -y gcc-9 autoconf2.69 && \ ln -sf /usr/bin/gcc-9 /usr/bin/gcc && \ ln -sf /usr/bin/autoconf2.69 /usr/bin/autoconf; \ - if [ "$TARGETARCH" = arm64 ] && [ "$BUILDARCH" != arm64 ]; then \ - apt-get install -y gcc-9-aarch64-linux-gnu && \ - ln -sf /usr/bin/aarch64-linux-gnu-gcc-9 /usr/bin/aarch64-linux-gnu-gcc; \ - fi \ fi ARG CFLAGS="-g -O2 -fdebug-prefix-map=/=. -fstack-protector-strong -Wformat -Werror=format-security" @@ -54,9 +41,6 @@ ARG CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" ARG LDFLAGS="-Wl,-Bsymbolic-functions -Wl,-z,relro" ARG ERLC_USE_SERVER=false RUN ./otp_build autoconf -RUN if [ "$TARGETARCH" = arm64 ]; then \ - ./configure --enable-bootstrap-only && make -j$(nproc); \ - fi RUN libssl_version=$(dpkg-query --showformat='${Version}' --show libssl-dev); \ STATIC_OPENSSL=$(dpkg --compare-versions "$erlang_version" lt 20 || (dpkg --compare-versions "$erlang_version" lt 24.2 && dpkg --compare-versions "$libssl_version" ge 3) && echo y); \ ./configure erl_xcomp_sysroot=/ \ @@ -77,21 +61,18 @@ RUN libssl_version=$(dpkg-query --showformat='${Version}' --show libssl-dev); \ --without-eunit \ --with-ssl-rpath=no \ --with-ssl \ - $([ "$TARGETARCH" = arm64 ] && echo "--host=aarch64-linux-gnu --build=$BUILDARCH-linux-gnu") \ $([ "$STATIC_OPENSSL" = y ] && echo "--with-ssl=/usr/local --disable-dynamic-ssl-lib" || echo --enable-dynamic-ssl-lib) && \ make -j$(nproc) && \ make install DESTDIR=/tmp/install && \ find /tmp/install -type d -name examples | xargs rm -r && \ - find /tmp/install -type f -executable -exec $([ "$TARGETARCH" = arm64 ] && echo aarch64-linux-gnu-)strip {} \;; + find /tmp/install -type f -executable -exec strip {} \;; # when cross compiling the target version of strip is required ARG erlang_iteration=1 -RUN readelf=$([ "$TARGETARCH" = arm64 ] && echo aarch64-linux-gnu-)readelf; \ - fpm -s dir -t deb \ +RUN fpm -s dir -t deb \ --chdir /tmp/install \ --name esl-erlang \ --version $erlang_version \ - --architecture $TARGETARCH \ --epoch 1 \ --iteration $erlang_iteration \ --maintainer "CloudAMQP " \ @@ -100,7 +81,7 @@ RUN readelf=$([ "$TARGETARCH" = arm64 ] && echo aarch64-linux-gnu-)readelf; \ --url "https://erlang.org" \ --license "Apache 2.0" \ --depends "procps" \ - --depends "$($readelf -d $(find /tmp/install/usr -name beam.smp) | awk '/NEEDED/{gsub(/[\[\]]/, "");print $5}' | xargs dpkg -S | cut -d: -f1 | sort -u | paste -sd,)" \ + --depends "$(readelf -d $(find /tmp/install/usr -name beam.smp) | awk '/NEEDED/{gsub(/[\[\]]/, "");print $5}' | xargs dpkg -S | cut -d: -f1 | sort -u | paste -sd,)" \ --conflicts "erlang-asn1,erlang-base,erlang-base-hipe,erlang-common-test,erlang-corba,erlang-crypto,erlang-debugger,erlang-dev,erlang-dialyzer,erlang-diameter,erlang-doc,erlang-edoc,erlang-eldap,erlang-erl-docgen,erlang-et,erlang-eunit,erlang-examples,erlang-ftp,erlang-ic,erlang-ic-java,erlang-inets,erlang-inviso,erlang-jinterface,erlang-manpages,erlang-megaco,erlang-mnesia,erlang-mode,erlang-nox,erlang-observer,erlang-odbc,erlang-os-mon,erlang-parsetools,erlang-percept,erlang-public-key,erlang-reltool,erlang-runtime-tools,erlang-snmp,erlang-src,erlang-ssh,erlang-ssl,erlang-syntax-tools,erlang-tftp,erlang-tools,erlang-webtool,erlang-wx,erlang-xmerl" #RUN apt-get install -y lintian diff --git a/bin/build-and-upload b/bin/build-and-upload new file mode 100755 index 0000000..ef7ac79 --- /dev/null +++ b/bin/build-and-upload @@ -0,0 +1,20 @@ +#!/usr/bin/env ruby +require_relative "../lib/packagecloud" + +if ARGV.size != 3 + abort "#{File.basename $PROGRAM_NAME} build-and-upload " +end + +version, image, platform = ARGV.shift(3) +system("depot", "build", + "--platform", "linux/#{platform}", + "--build-arg", "erlang_version=#{version}", + "--build-arg", "image=#{image.sub('/', ':')}", + "--output", ".", + ".", exception: true) + +packagecloud = Packagecloud.new +File.open("esl-erlang_#{version}-1_#{platform}.deb") do |file| + packagecloud.upload(image.sub(":", "/"), file) + File.unlink(file) +end diff --git a/bin/missing-versions b/bin/missing-versions new file mode 100755 index 0000000..ae676ef --- /dev/null +++ b/bin/missing-versions @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +require_relative "../lib/github" +require_relative "../lib/packagecloud" + +DISTS = %w[ubuntu/jammy ubuntu/focal].freeze +PLATFORMS = %w[amd64 arm64].freeze + +packagecloud = Packagecloud.new +github = Github.new + +missing = [] +github.releases do |r| + next if r["prerelease"] + next if r["draft"] + next if r["tag_name"].include? "-rc" + version = r["tag_name"].sub("OTP-", "") + DISTS.each do |dist| + PLATFORMS.each do |platform| + filename = "esl-erlang_#{version}-1_#{platform}.deb" + next if packagecloud.exists? dist, filename + image = dist.sub("/", ":") + missing << { version:, image:, platform: } + end + end +end + +# Output for Github Action +JSON.dump(missing.take(256), $stdout) + +# Github Actions support maximum 256 jobs +warn "Result truncated to 256, actual missing versions: #{to_build.size}" if to_build.size > 256 diff --git a/depot.json b/depot.json new file mode 100644 index 0000000..3e9c8f1 --- /dev/null +++ b/depot.json @@ -0,0 +1 @@ +{"id":"srqn8st7l7"} diff --git a/lib/github.rb b/lib/github.rb new file mode 100644 index 0000000..8677479 --- /dev/null +++ b/lib/github.rb @@ -0,0 +1,20 @@ +require "net/http" +require "json" + +class Github + def initialize(token = ENV.fetch("GITHUB_TOKEN")) + @auth = { Authorization: "Bearer #{token}" } + end + + def releases(&blk) + Net::HTTP.start("api.github.com", use_ssl: true) do |api| + 1.upto(10).each do |page| + resp = api.get("/repos/erlang/otp/releases?per_page=100&&page=#{page}", @auth) + raise "Unexpected response: #{resp} #{resp.body}" unless Net::HTTPOK === resp + releases = JSON.parse(resp.body) + break if releases.empty? + releases.each(&blk) + end + end + end +end diff --git a/lib/packagecloud.rb b/lib/packagecloud.rb new file mode 100644 index 0000000..98b48bc --- /dev/null +++ b/lib/packagecloud.rb @@ -0,0 +1,72 @@ +require "net/http" +require "json" + +class Packagecloud + def initialize(token = ENV.fetch("PACKAGECLOUD_TOKEN")) + @http = Net::HTTP.start("packagecloud.io", use_ssl: true) + @token = token + @packages = packages + @distributions = distributions + end + + def exists?(dist_name, name) + @packages.any? { |p| p["filename"] == name && p["distro_version"] == dist_name } + end + + def upload(dist_name, file) + puts "Uploading #{File.basename file.path} (#{(file.size / 1024.0**2).round(1)} MB)" + file.rewind + request = Net::HTTP::Post.new("/api/v1/repos/cloudamqp/erlang/packages.json") + request.basic_auth(@token, "") + form_data = [["package[distro_version_id]", dist_id(dist_name).to_s], + ["package[package_file]", file]] + request.set_form form_data, "multipart/form-data" + response = @http.request(request) + case response + when Net::HTTPCreated + package = JSON.parse(response.body) + puts "#{package['filename']} for #{package['distro_version']} uploaded" + else raise "Unexpected response: #{response} #{response.body}" + end + end + + def dist_id(name) + @distributions.each_value do |type| + type.each do |dist| + dist["versions"].each do |v| + dist_name = "#{dist['index_name']}/#{v['index_name']}" + return v["id"] if dist_name == name + end + end + end + end + + private + + def packages + packages = [] + 1.upto(10) do |page| + path = "/api/v1/repos/cloudamqp/erlang/packages.json?per_page=250&page=#{page}" + request = Net::HTTP::Get.new(path) + request.basic_auth(@token, "") + resp = @http.request request + raise "Unexpected response: #{resp} #{resp.body}" unless Net::HTTPOK === resp + + data = JSON.parse(resp.body) + break if data.empty? + packages.concat data + end + packages + end + + def distributions + request = Net::HTTP::Get.new("/api/v1/distributions.json") + request.basic_auth(@token, "") + resp = @http.request request + raise "Unexpected response: #{resp} #{resp.body}" unless Net::HTTPOK === resp + + JSON.parse resp.body + end +end + +