From 3f2ecf39cfd1fedb37657955ce363c270651490f Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Tue, 14 Jan 2025 14:57:54 +0000 Subject: [PATCH 1/9] feat: Add reproducible build profile and Dockerfile --- Cargo.toml | 15 +++++++++++++++ Dockerfile | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 Dockerfile diff --git a/Cargo.toml b/Cargo.toml index e562ee2..b64c8b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,3 +66,18 @@ rbuilder = { git = "https://github.com/flashbots/rbuilder.git", rev = "945f03959 [build-dependencies] built = { version = "0.7.1", features = ["git2", "chrono"] } tonic-build = "0.8" + +[profile.release] +codegen-units = 16 +debug = "line-tables-only" +lto = "thin" +opt-level = 3 +panic = "unwind" +strip = true + +[profile.reproducible] +inherits = "release" +debug = false +panic = "abort" +codegen-units = 1 +overflow-checks = true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..33c3db4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af2672370559d7e854653c66d AS builder + +ARG BUILD_PROFILE=reproducible +ENV BUILD_PROFILE=$BUILD_PROFILE + +# Extra Cargo flags +ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $(pwd)=." +ENV RUSTFLAGS="$RUSTFLAGS" + +# Extra Cargo features +ARG FEATURES="" +ENV FEATURES=$FEATURES + +RUN apt-get update && apt-get install -y \ + libclang-dev=1:11.0-51+nmu5 \ + protobuf-compiler=3.12.4-1+deb11u1 + +# Clone the repository at the specific branch +WORKDIR /app +COPY ./ /app + +# Get the latest commit timestamp and set SOURCE_DATE_EPOCH +RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ + echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment + +# Set environment variables for reproducibility +ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ + CARGO_INCREMENTAL=0 \ + LC_ALL=C \ + TZ=UTC \ + RUSTFLAGS="${RUSTFLAGS}" + +# Build the project with the reproducible settings +RUN . /etc/environment && \ + cargo build --features "${FEATURES}" --profile "${BUILD_PROFILE}" --locked --target x86_64-unknown-linux-gnu + +RUN . /etc/environment && mv /app/target/x86_64-unknown-linux-gnu/"${BUILD_PROFILE}"/rbuilder /rbuilder + +FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a +COPY --from=builder /rbuilder /rbuilder +ENTRYPOINT [ "/rbuilder" ] From c00dfff746fa24a1d4d7ed1a6ff81ac326afd1a5 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Thu, 16 Jan 2025 10:16:00 +0000 Subject: [PATCH 2/9] remove the extra Cargo.toml profiles --- Cargo.toml | 15 --------------- Dockerfile | 2 +- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b64c8b1..e562ee2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,18 +66,3 @@ rbuilder = { git = "https://github.com/flashbots/rbuilder.git", rev = "945f03959 [build-dependencies] built = { version = "0.7.1", features = ["git2", "chrono"] } tonic-build = "0.8" - -[profile.release] -codegen-units = 16 -debug = "line-tables-only" -lto = "thin" -opt-level = 3 -panic = "unwind" -strip = true - -[profile.reproducible] -inherits = "release" -debug = false -panic = "abort" -codegen-units = 1 -overflow-checks = true diff --git a/Dockerfile b/Dockerfile index 33c3db4..60d1832 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af2672370559d7e854653c66d AS builder -ARG BUILD_PROFILE=reproducible +ARG BUILD_PROFILE=release ENV BUILD_PROFILE=$BUILD_PROFILE # Extra Cargo flags From 655cab8f33da155a20c70e14e070be67bfcc3eb5 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 17 Jan 2025 12:50:06 +0000 Subject: [PATCH 3/9] Add comments to the rust flags --- Dockerfile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 60d1832..f9baae1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,17 @@ FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af26723705 ARG BUILD_PROFILE=release ENV BUILD_PROFILE=$BUILD_PROFILE -# Extra Cargo flags -ARG RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-Wl,--build-id=none -Clink-arg=-static-libgcc -C metadata='' --remap-path-prefix $(pwd)=." +# RUSTFLAGS breakdown: +# -C target-feature=+crt-static -> Statically link the C runtime library for standalone binaries +# -C link-arg=-Wl,--build-id=none -> Remove build ID from binary for reproducibility +# -Clink-arg=-static-libgcc -> Statically link against libgcc +# -C metadata='' -> Remove metadata hash from symbol names for reproducible builds +# --remap-path-prefix $(pwd)=. -> Replace absolute paths with '.' in debug info +ARG RUSTFLAGS="-C target-feature=+crt-static \ + -C link-arg=-Wl,--build-id=none \ + -Clink-arg=-static-libgcc \ + -C metadata='' \ + --remap-path-prefix $(pwd)=." ENV RUSTFLAGS="$RUSTFLAGS" # Extra Cargo features From 16b2904e0f35a9a296427b1343080673dcf85328 Mon Sep 17 00:00:00 2001 From: Moe Mahhouk Date: Fri, 17 Jan 2025 15:22:23 +0100 Subject: [PATCH 4/9] feat: add extra job for the reproducible docker build in the CI (#43) * feat: add extra job for the reproducible docker build in the CI * remove the extra Cargo.toml profiles * add comments to the rust flags and move them into Makefile * Add comments to the rust flags --- .github/workflows/release.yaml | 63 +++++++++++++++++++++++++++++++++- Makefile | 41 ++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4353aa1..d726472 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,6 +11,11 @@ on: required: false type: boolean default: true + build-docker: + description: 'Build Docker' + required: false + type: boolean + default: true draft-release: description: 'Draft Release' required: false @@ -79,6 +84,8 @@ jobs: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - uses: actions/checkout@v4 # must install git before checkout and set safe.directory after checkout because of container + with: + fetch-depth: 0 - name: Prepare filename run: echo "OUTPUT_FILENAME=rbuilder-${VERSION}-${{ matrix.configs.target }}" >> $GITHUB_ENV @@ -87,7 +94,7 @@ jobs: run: | git config --global --add safe.directory "$(pwd)" . $HOME/.cargo/env - cargo build --release --target ${{ matrix.configs.target }} + make build-reproducible TARGET=${{ matrix.configs.target }} ./target/${{ matrix.configs.target }}/release/rbuilder version - name: Upload artifact @@ -96,6 +103,60 @@ jobs: name: ${{ env.OUTPUT_FILENAME }} path: target/${{ matrix.configs.target }}/release/rbuilder + build-docker: + name: Build and publish Docker image + if: ${{ github.event.inputs.build-docker == 'true' || github.event_name == 'push'}} + needs: extract-version + runs-on: warp-ubuntu-latest-x64-16x + env: + VERSION: ${{ needs.extract-version.outputs.VERSION }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker metadata + uses: docker/metadata-action@v5 + id: meta + with: + images: ghcr.io/${{ github.repository }} + labels: org.opencontainers.image.source=${{ github.repositoryUrl }} + tags: | + type=sha + type=semver,pattern={{version}},value=${{ env.VERSION }} + type=semver,pattern={{major}}.{{minor}},value=${{ env.VERSION }} + type=semver,pattern={{major}},value=${{ env.VERSION }} + type=raw,value=latest,enable=${{ !contains(env.VERSION, '-') }} + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILD_PROFILE=release + draft-release: name: Draft release if: ${{ github.event.inputs.draft-release == 'true' || github.event_name == 'push'}} # when manually triggered or version tagged diff --git a/Makefile b/Makefile index 627e31e..1995fba 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,47 @@ clean: ## Clean up build: ## Build static binary for x86_64 cargo build --release --target x86_64-unknown-linux-gnu +# Environment variables for reproducible builds +# Initialize RUSTFLAGS +RUST_BUILD_FLAGS = + +# Statically link the C runtime library for standalone binaries +RUST_BUILD_FLAGS += -C target-feature=+crt-static + +# Remove build ID from the binary to ensure reproducibility across builds +RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none + +# Statically link against libgcc to remove runtime dependencies +RUST_BUILD_FLAGS += -C link-arg=-static-libgcc + +# Remove metadata hash from symbol names to ensure reproducible builds +RUST_BUILD_FLAGS += -C metadata='' + +# Set timestamp from last git commit for reproducible builds +SOURCE_DATE ?= $(shell git log -1 --pretty=%ct) + +# Disable incremental compilation to avoid non-deterministic artifacts +CARGO_INCREMENTAL_VAL = 0 + +# Set C locale for consistent string handling and sorting +LOCALE_VAL = C + +# Set UTC timezone for consistent time handling across builds +TZ_VAL = UTC + +# Set the target for the build, default to x86_64 +TARGET ?= x86_64-unknown-linux-gnu + +.PHONY: build-reproducible +build-reproducible: ## Build reproducible static binary for x86_64 + # Set timestamp from last git commit for reproducible builds + SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \ + CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ + LC_ALL=${LOCALE_VAL} \ + TZ=${TZ_VAL} \ + cargo build --release --locked --target $(TARGET) + .PHONY: docker-image docker-image: ## Build a rbuilder Docker image docker build --platform linux/amd64 . -t rbuilder From c977d1a03ebfe7da6de4593f14e52c89039dcea3 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 17 Jan 2025 14:52:35 +0000 Subject: [PATCH 5/9] refactor the comments position --- Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index f9baae1..2626daa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,17 @@ FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af26723705 ARG BUILD_PROFILE=release ENV BUILD_PROFILE=$BUILD_PROFILE -# RUSTFLAGS breakdown: -# -C target-feature=+crt-static -> Statically link the C runtime library for standalone binaries -# -C link-arg=-Wl,--build-id=none -> Remove build ID from binary for reproducibility -# -Clink-arg=-static-libgcc -> Statically link against libgcc -# -C metadata='' -> Remove metadata hash from symbol names for reproducible builds -# --remap-path-prefix $(pwd)=. -> Replace absolute paths with '.' in debug info -ARG RUSTFLAGS="-C target-feature=+crt-static \ - -C link-arg=-Wl,--build-id=none \ - -Clink-arg=-static-libgcc \ - -C metadata='' \ - --remap-path-prefix $(pwd)=." +ARG RUSTFLAGS="\ +# Statically link the C runtime library for standalone binaries +-C target-feature=+crt-static \ +# Remove build ID from binary for reproducibility +-C link-arg=-Wl,--build-id=none \ +# Statically link against libgcc +-Clink-arg=-static-libgcc \ +# Remove metadata hash from symbol names for reproducible builds +-C metadata='' \ +# Replace absolute paths with '.' in debug info +--remap-path-prefix $(pwd)=." ENV RUSTFLAGS="$RUSTFLAGS" # Extra Cargo features From 3a6cf9a90a2f9c9265d2d97bde7d8cf2ee8eaa42 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 17 Jan 2025 16:01:03 +0000 Subject: [PATCH 6/9] chore: re-use the build logic and setup from the makefile --- Dockerfile | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2626daa..4001657 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,23 +3,6 @@ FROM rust:1.82-bullseye@sha256:c42c8ca762560c182ba30edda0e0d71a8604040af26723705 ARG BUILD_PROFILE=release ENV BUILD_PROFILE=$BUILD_PROFILE -ARG RUSTFLAGS="\ -# Statically link the C runtime library for standalone binaries --C target-feature=+crt-static \ -# Remove build ID from binary for reproducibility --C link-arg=-Wl,--build-id=none \ -# Statically link against libgcc --Clink-arg=-static-libgcc \ -# Remove metadata hash from symbol names for reproducible builds --C metadata='' \ -# Replace absolute paths with '.' in debug info ---remap-path-prefix $(pwd)=." -ENV RUSTFLAGS="$RUSTFLAGS" - -# Extra Cargo features -ARG FEATURES="" -ENV FEATURES=$FEATURES - RUN apt-get update && apt-get install -y \ libclang-dev=1:11.0-51+nmu5 \ protobuf-compiler=3.12.4-1+deb11u1 @@ -28,22 +11,10 @@ RUN apt-get update && apt-get install -y \ WORKDIR /app COPY ./ /app -# Get the latest commit timestamp and set SOURCE_DATE_EPOCH -RUN SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \ - echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH" >> /etc/environment - -# Set environment variables for reproducibility -ENV SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH \ - CARGO_INCREMENTAL=0 \ - LC_ALL=C \ - TZ=UTC \ - RUSTFLAGS="${RUSTFLAGS}" - # Build the project with the reproducible settings -RUN . /etc/environment && \ - cargo build --features "${FEATURES}" --profile "${BUILD_PROFILE}" --locked --target x86_64-unknown-linux-gnu +RUN make build-reproducible -RUN . /etc/environment && mv /app/target/x86_64-unknown-linux-gnu/"${BUILD_PROFILE}"/rbuilder /rbuilder +RUN mv /app/target/x86_64-unknown-linux-gnu/"${BUILD_PROFILE}"/rbuilder /rbuilder FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a COPY --from=builder /rbuilder /rbuilder From 1977afb939414f3068b1c373d3f94e2d4f6914f2 Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 17 Jan 2025 16:35:51 +0000 Subject: [PATCH 7/9] remove unnecessary flags --- Makefile | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/Makefile b/Makefile index 1995fba..8651bac 100644 --- a/Makefile +++ b/Makefile @@ -25,45 +25,15 @@ clean: ## Clean up build: ## Build static binary for x86_64 cargo build --release --target x86_64-unknown-linux-gnu -# Environment variables for reproducible builds -# Initialize RUSTFLAGS -RUST_BUILD_FLAGS = - -# Statically link the C runtime library for standalone binaries -RUST_BUILD_FLAGS += -C target-feature=+crt-static - -# Remove build ID from the binary to ensure reproducibility across builds -RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none - -# Statically link against libgcc to remove runtime dependencies -RUST_BUILD_FLAGS += -C link-arg=-static-libgcc - -# Remove metadata hash from symbol names to ensure reproducible builds -RUST_BUILD_FLAGS += -C metadata='' - # Set timestamp from last git commit for reproducible builds SOURCE_DATE ?= $(shell git log -1 --pretty=%ct) -# Disable incremental compilation to avoid non-deterministic artifacts -CARGO_INCREMENTAL_VAL = 0 - -# Set C locale for consistent string handling and sorting -LOCALE_VAL = C - -# Set UTC timezone for consistent time handling across builds -TZ_VAL = UTC - # Set the target for the build, default to x86_64 TARGET ?= x86_64-unknown-linux-gnu .PHONY: build-reproducible build-reproducible: ## Build reproducible static binary for x86_64 - # Set timestamp from last git commit for reproducible builds SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ - RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \ - CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ - LC_ALL=${LOCALE_VAL} \ - TZ=${TZ_VAL} \ cargo build --release --locked --target $(TARGET) .PHONY: docker-image From c3a0a7fff1ea656984d0400ed02ec5c75e0dd23e Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 17 Jan 2025 17:04:21 +0000 Subject: [PATCH 8/9] Revert "remove unnecessary flags" This reverts commit 1977afb939414f3068b1c373d3f94e2d4f6914f2. --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Makefile b/Makefile index 8651bac..1995fba 100644 --- a/Makefile +++ b/Makefile @@ -25,15 +25,45 @@ clean: ## Clean up build: ## Build static binary for x86_64 cargo build --release --target x86_64-unknown-linux-gnu +# Environment variables for reproducible builds +# Initialize RUSTFLAGS +RUST_BUILD_FLAGS = + +# Statically link the C runtime library for standalone binaries +RUST_BUILD_FLAGS += -C target-feature=+crt-static + +# Remove build ID from the binary to ensure reproducibility across builds +RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none + +# Statically link against libgcc to remove runtime dependencies +RUST_BUILD_FLAGS += -C link-arg=-static-libgcc + +# Remove metadata hash from symbol names to ensure reproducible builds +RUST_BUILD_FLAGS += -C metadata='' + # Set timestamp from last git commit for reproducible builds SOURCE_DATE ?= $(shell git log -1 --pretty=%ct) +# Disable incremental compilation to avoid non-deterministic artifacts +CARGO_INCREMENTAL_VAL = 0 + +# Set C locale for consistent string handling and sorting +LOCALE_VAL = C + +# Set UTC timezone for consistent time handling across builds +TZ_VAL = UTC + # Set the target for the build, default to x86_64 TARGET ?= x86_64-unknown-linux-gnu .PHONY: build-reproducible build-reproducible: ## Build reproducible static binary for x86_64 + # Set timestamp from last git commit for reproducible builds SOURCE_DATE_EPOCH=$(SOURCE_DATE) \ + RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \ + CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \ + LC_ALL=${LOCALE_VAL} \ + TZ=${TZ_VAL} \ cargo build --release --locked --target $(TARGET) .PHONY: docker-image From f3546b2cbc872f00284ffd2f58fe8341b6c09ebb Mon Sep 17 00:00:00 2001 From: MoeMahhouk Date: Fri, 17 Jan 2025 17:24:23 +0000 Subject: [PATCH 9/9] remove static linking of libgcc --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index 1995fba..6ee25cb 100644 --- a/Makefile +++ b/Makefile @@ -29,15 +29,9 @@ build: ## Build static binary for x86_64 # Initialize RUSTFLAGS RUST_BUILD_FLAGS = -# Statically link the C runtime library for standalone binaries -RUST_BUILD_FLAGS += -C target-feature=+crt-static - # Remove build ID from the binary to ensure reproducibility across builds RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none -# Statically link against libgcc to remove runtime dependencies -RUST_BUILD_FLAGS += -C link-arg=-static-libgcc - # Remove metadata hash from symbol names to ensure reproducible builds RUST_BUILD_FLAGS += -C metadata=''