build: create Docker images automatically

nagilooh · Jun 24, 2024 · d46a69a · d46a69a
1 parent d5efb58
commit d46a69a
Show file tree

Hide file tree

Showing 11 changed files with 241 additions and 121 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,6 +13,8 @@ on:
 jobs:
   build:
     name: Build
+    permissions:
+      contents: read
     strategy:
       matrix:
         os:
@@ -32,10 +34,6 @@ jobs:
           if [ "${SONAR_TOKEN}" != '' ]; then
             echo 'is_SONAR_TOKEN_set=true' >> $GITHUB_OUTPUT
           fi
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: ${{ !steps.check-secret.outputs.is_SONAR_TOKEN_set && 1 || 0 }}  # Shallow clones should be disabled for a better relevancy of SonarCloud analysis
       - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
@@ -56,6 +54,10 @@ jobs:
             ~/.sonar/cache
           key: ${{ matrix.os }}-sonar
           restore-keys: ${{ matrix.os }}-sonar
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: ${{ !steps.check-secret.outputs.is_SONAR_TOKEN_set && 1 || 0 }}  # Shallow clones should be disabled for a better relevancy of SonarCloud analysis
       - name: Cache node distribution
         uses: actions/cache@v4
         with:
@@ -81,15 +83,15 @@ jobs:
         run: |
           ./gradlew sonar -Pci --info --stacktrace --max-workers 4 --no-daemon
       - name: Build signed Maven repository
-        if: ${{ matrix.os == 'ubuntu-latest' && github.event_name == 'push' && github.repository == 'graphs4value/refinery' }}
+        if: ${{ matrix.os == 'ubuntu-latest' && github.event_name == 'push' && github.repository_owner == 'graphs4value' }}
         env:
           PGP_KEY: ${{ secrets.PGP_KEY }}
           PGP_KEY_ID: ${{ secrets.PGP_KEY_ID }}
           PGP_PASSWORD: ${{ secrets.PGP_PASSWORD }}
         run: |
           ./gradlew mavenRepositoryTar -Pci -PforceSign --info --stacktrace --max-workers 4 --no-daemon
       - name: Build unsigned Maven repository
-        if: ${{ matrix.os == 'ubuntu-latest' && (github.event_name != 'push' || github.repository != 'graphs4value/refinery') }}
+        if: ${{ matrix.os == 'ubuntu-latest' && (github.event_name != 'push' || github.repository_owner != 'graphs4value') }}
         run: |
           ./gradlew mavenRepositoryTar -Pci --info --stacktrace --max-workers 4 --no-daemon
       - name: Upload Maven repository artifact
@@ -99,6 +101,14 @@ jobs:
           name: maven-repository-tar
           path: build/refinery-maven-repository.tar
           compression-level: 0
+      - name: Upload application artifacts
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: distributions-tar
+          path: subprojects/**/build/distributions/*.tar
+          compression-level: 0
+          retention-days: 5  # No need to preserve for long, since they are uploaded to GHCR
       - name: Upload site artifact
         if: ${{ matrix.os == 'ubuntu-latest' }}
         uses: actions/upload-artifact@v4
@@ -108,9 +118,12 @@ jobs:
           compression-level: 0
   reuse-check:
     name: REUSE Compliance Check
+    permissions:
+      contents: read
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
       - name: REUSE Compliance Check
         uses: fsfe/reuse-action@a46482ca367aef4454a87620aa37c2be4b2f8106
         with:
@@ -140,25 +153,79 @@ jobs:
           git_config_global: true
           git_user_signingkey: true
           git_commit_gpgsign: true
-      - name: Commit and push to graphs4value.github.io
-        env:
-          GH_PAGES_TOKEN: ${{ secrets.GH_PAGES_TOKEN }}
-          GITHUB_REPOSITORY: ${{ github.sha }}
-          GITHUB_SHA: ${{ github.sha }}
+      - name: Create empty git repository
         run: |
           mkdir graphs4value.github.io
-          pushd graphs4value.github.io
+          cd graphs4value.github.io
           git config --global init.defaultBranch main
           git config --global user.name "Graphs4Value bot"
           git config --global user.email "[email protected]"
           git init
-          git remote add origin "https://x-access-token:${GH_PAGES_TOKEN}@github.com/graphs4value/graphs4value.github.io.git"
+      - name: Extract site artifact
+        working-directory: ./graphs4value.github.io
+        run: |
           unzip ../site-zip/refinery-docs.zip
+      - name: Extract Maven repository artifact
+        working-directory: ./graphs4value.github.io
+        run: |
           mkdir -p maven/snapshots
-          pushd maven/snapshots
-          tar xf ../../../maven-repository-tar/refinery-maven-repository.tar
-          popd
+          cd maven/snapshots
+          tar -xvf ../../../maven-repository-tar/refinery-maven-repository.tar
+      - name: Commit and push to graphs4value.github.io
+        working-directory: ./graphs4value.github.io
+        env:
+          GH_PAGES_TOKEN: ${{ secrets.GH_PAGES_TOKEN }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: |
+          git remote add origin "https://x-access-token:${GH_PAGES_TOKEN}@github.com/graphs4value/graphs4value.github.io.git"
           git add .
           git commit -S -m "Update from https://github.com/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}"
           git push --force origin main
-          popd
+  docker-build:
+    name: Build Docker images
+    needs: build
+    permissions:
+      packages: write
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
+        with:
+          platforms: arm64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb
+        with:
+          platforms: linux/amd64,linux/arm64
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Download application artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: distributions-tar
+          path: subprojects
+      - name: Extract application artifacts
+        working-directory: ./docker
+        run: |
+          ./prepare_context.sh
+      - name: Bake images
+        working-directory: ./docker
+        run: |
+          ./bake.sh false --set '*.cache-from=gha' --set '*.cache-to=type=gha,mode=max'
+      - name: Log in to GitHub Container registry
+        if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'graphs4value/refinery' }}
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload images to GitHub Container registry
+        if: ${{ github.event_name == 'push' && github.ref_name == 'main' && github.repository == 'graphs4value/refinery' }}
+        working-directory: ./docker
+        run: |
+          ./bake.sh true --set '*.cache-from=gha' --set '*.cache-to=type=gha,mode=max'
+      - name: Delete application artifacts
+        uses: geekyeggo/delete-artifact@24928e75e6e6590170563b8ddae9fac674508aa1
+        with:
+          name: distributions-tar
diff --git a/docker/.gitignore b/docker/.gitignore
@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2024 The Refinery Authors
+#
+# SPDX-License-Identifier: CC0-1.0
+
+context/extracted
diff --git a/docker/bake.sh b/docker/bake.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# SPDX-FileCopyrightText: 2024 The Refinery Authors <https://refinery.tools/>
+#
+# SPDX-License-Identifier: EPL-2.0
+
+set -euo pipefail
+
+REFINERY_VERSION="$(./get_version.sh)"
+export REFINERY_VERSION
+
+export REFINERY_PUSH="${1-false}"
+
+export SOURCE_DATE_EPOCH=0
+
+cd context
+exec docker buildx bake -f docker-bake.hcl "${@:2}"
diff --git a/docker/build.sh b/docker/build.sh
@@ -1,55 +1,13 @@
 #!/usr/bin/env bash
 
-# SPDX-FileCopyrightText: 2023-2024 The Refinery Authors <https://refinery.tools/>
+# SPDX-FileCopyrightText: 2024 The Refinery Authors <https://refinery.tools/>
 #
 # SPDX-License-Identifier: EPL-2.0
 
 set -euo pipefail
 
 (cd .. && ./gradlew distTar)
 
-refinery_version="$(grep '^version=' ../gradle.properties | cut -d'=' -f2)"
-cli_distribution_name="refinery-generator-cli-${refinery_version}"
-web_distribution_name="refinery-language-web-${refinery_version}"
+./prepare_context.sh
 
-rm -rf "${cli_distribution_name}" "${web_distribution_name}" {cli,web}_dist \
-    {cli,web}_{,app_}lib common_{,amd64_,arm64_}lib {cli,web}_{amd64,arm64}_bin
-
-tar -xf "../subprojects/generator-cli/build/distributions/${cli_distribution_name}.tar"
-mv "${cli_distribution_name}" cli_dist
-tar -xf "../subprojects/language-web/build/distributions/${web_distribution_name}.tar"
-mv "${web_distribution_name}" web_dist
-mkdir -p {cli,web}_{,app_}lib common_{,amd64_,arm64_}lib {cli,web}_{amd64,arm64}_bin
-
-# Our application itself is very small, so it will get added as the last layer
-# of both containers.
-mv cli_dist/lib/refinery-* cli_app_lib
-mv web_dist/lib/refinery-* web_app_lib
-
-for i in cli_dist/lib/*; do
-    j="web${i#cli}"
-    if [[ -f "$j" ]]; then
-        mv "$i" "common_lib${i#cli_dist/lib}"
-        rm "$j"
-    fi
-done
-
-# Move architecture-specific jars to their repsective directories.
-mv common_lib/ortools-linux-x86-64-*.jar common_amd64_lib
-mv common_lib/ortools-linux-aarch64-*.jar common_arm64_lib
-rm common_lib/ortools-{darwin,win32}-*.jar
-# Move the applications jars for the dependencies into a separate Docker layer
-# to enable faster updates.
-mv cli_dist/lib/* cli_lib
-mv web_dist/lib/* web_lib
-# Omit references to jars not present for the current architecture from the
-# startup scripts.
-sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' cli_dist/bin/refinery-generator-cli > cli_amd64_bin/refinery-generator-cli
-sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' cli_dist/bin/refinery-generator-cli > cli_arm64_bin/refinery-generator-cli
-chmod a+x cli_{amd64,arm64}_bin/refinery-generator-cli
-sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-aarch64\)[^:]\+\.jar//g' web_dist/bin/refinery-language-web > web_amd64_bin/refinery-language-web
-sed 's/:\$APP_HOME\/lib\/ortools-\(darwin\|win32\|linux-x86-64\)[^:]\+\.jar//g' web_dist/bin/refinery-language-web > web_arm64_bin/refinery-language-web
-chmod a+x web_{amd64,arm64}_bin/refinery-language-web
-rm -rf {cli,web}_dist
-
-REFINERY_VERSION="${refinery_version}" docker buildx bake -f docker-bake.hcl
+./bake.sh "${1-false}" "${@:2}"
diff --git a/docker/Dockerfile.base → docker/context/Dockerfile.base b/docker/Dockerfile.base → docker/context/Dockerfile.base
@@ -30,8 +30,8 @@ RUN apt-get update && \
     ln -sf /usr/bin/x86_64-linux-gnu-objcopy /usr/bin/objcopy && \
     rm -rf /var/lib/apt/lists/*
 
-FROM --platform=$BUILDPLATFORM jlink-$TARGETARCH-on-$BUILDARCH as jlink
-RUN jlink --no-header-files --no-man-pages --compress=2 \
+FROM --platform=$BUILDPLATFORM jlink-$TARGETARCH-on-$BUILDARCH AS jlink
+RUN jlink --no-header-files --no-man-pages \
     --module-path=/crossjdk/jmods --strip-debug --add-modules \
     java.base,java.logging,java.xml,jdk.zipfs \
     --output /jlink
@@ -44,15 +44,14 @@ RUN dnf install -y findutils && \
 COPY --link --from=jlink /jlink /usr/lib/java
 ENV JAVA_HOME="/usr/lib/java" PATH="/usr/lib/java/bin:${PATH}"
 # Layer with platform-independent dependencies, slow changing.
-ADD --link common_lib /app/lib
+ADD --link extracted/common_lib /app/lib
 
 FROM base AS base-amd64
 # Layer with platform-dependent dependencies, slow changing.
-ADD --link common_amd64_lib /app/lib
+ADD --link extracted/common_amd64_lib /app/lib
 
 FROM base AS base-arm64
 # Layer with platform-dependent dependencies, slow changing.
-ADD --link common_arm64_lib /app/lib
+ADD --link extracted/common_arm64_lib /app/lib
 
 FROM base-$TARGETARCH
-
diff --git a/docker/Dockerfile.cli → docker/context/Dockerfile.cli b/docker/Dockerfile.cli → docker/context/Dockerfile.cli
@@ -4,21 +4,21 @@
 
 FROM base AS cli-base
 # Layer with platform-dependent dependencies, slow changing.
-ADD --link cli_lib /app/lib
+ADD --link extracted/cli_lib /app/lib
 
 FROM cli-base AS cli-amd64
 # Layer with platform-dependent startup script containing references to all
 # dependency versions.
-ADD --link cli_amd64_bin /app/bin
+ADD --link extracted/cli_amd64_bin /app/bin
 
 FROM cli-base AS cli-arm64
 # Layer with platform-dependent startup script containing references to all
 # dependency versions.
-ADD --link cli_arm64_bin /app/bin
+ADD --link extracted/cli_arm64_bin /app/bin
 
 FROM cli-$TARGETARCH
 # Layer with platform-independent application jars.
-ADD --link cli_app_lib /app/lib
+ADD --link extracted/cli_app_lib /app/lib
 # Common settings added on top.
 ENV REFINERY_LIBRARY_PATH=/data
 USER 1000

diff --git a/docker/Dockerfile.web → docker/context/Dockerfile.web b/docker/Dockerfile.web → docker/context/Dockerfile.web
@@ -4,21 +4,21 @@
 
 FROM base AS web-base
 # Layer with platform-dependent dependencies, slow changing.
-ADD --link web_lib /app/lib
+ADD --link extracted/web_lib /app/lib
 
 FROM web-base AS web-amd64
 # Layer with platform-dependent startup script containing references to all
 # dependency versions.
-ADD --link web_amd64_bin /app/bin
+ADD --link extracted/web_amd64_bin /app/bin
 
 FROM web-base AS web-arm64
 # Layer with platform-dependent startup script containing references to all
 # dependency versions.
-ADD --link web_arm64_bin /app/bin
+ADD --link extracted/web_arm64_bin /app/bin
 
 FROM web-$TARGETARCH
 # Layer with platform-independent application jars.
-ADD --link web_app_lib /app/lib
+ADD --link extracted/web_app_lib /app/lib
 # Common settings added on top.
 ENV REFINERY_LISTEN_HOST=0.0.0.0 REFINERY_LISTEN_PORT=8888
 EXPOSE 8888

diff --git a/docker/context/docker-bake.hcl b/docker/context/docker-bake.hcl
@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2024 The Refinery Authors <https://refinery.tools/>
+#
+# SPDX-License-Identifier: EPL-2.0
+
+variable "REFINERY_VERSION" {
+  default = ""
+}
+
+variable "REFINERY_PUSH" {
+  default = "false"
+}
+
+group "default" {
+  targets = ["cli", "web"]
+}
+
+target "base" {
+  dockerfile = "Dockerfile.base"
+  platforms = ["linux/amd64", "linux/arm64"]
+  output = ["type=cacheonly"]
+}
+
+target "cli" {
+  dockerfile = "Dockerfile.cli"
+  platforms = ["linux/amd64", "linux/arm64"]
+  output = [
+    "type=image,push=${REFINERY_PUSH},\"name=ghcr.io/graphs4value/refinery-cli:${REFINERY_VERSION},ghcr.io/graphs4value/refinery-cli:latest\",annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,\"annotation-index.org.opencontainers.image.description=Command line interface for Refinery, an efficient graph solver for generating well-formed models\",annotation-index.org.opencontainers.image.licenses=EPL-2.0"
+  ]
+  contexts = {
+    base = "target:base"
+  }
+}
+
+target "web" {
+  dockerfile = "Dockerfile.web"
+  platforms = ["linux/amd64", "linux/arm64"]
+  output = [
+    "type=image,push=${REFINERY_PUSH},\"name=ghcr.io/graphs4value/refinery:${REFINERY_VERSION},ghcr.io/graphs4value/refinery:latest\",annotation-index.org.opencontainers.image.source=https://github.com/graphs4value/refinery,annotation-index.org.opencontainers.image.description=Refinery: an efficient graph solver for generating well-formed models,annotation-index.org.opencontainers.image.licenses=EPL-2.0"
+  ]
+  contexts = {
+    base = "target:base"
+  }
+}