feat: set up codspeed (paradigmxyz#13372)

Author: DaniPopes

.github/scripts/codspeed-build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
+
+cmd=(cargo codspeed build --profile profiling)
+excludes=(
+    # Unnecessary
+    --exclude reth-libmdbx
+    # Build is too slow
+    --exclude reth-network
+    # Built separately 
+    --exclude reth-transaction-pool
+    # TODO: some benchmarks panic: https://github.com/paradigmxyz/reth/actions/runs/12307046814/job/34349955788
+    --exclude reth-db
+    --exclude reth-trie-parallel
+    --exclude reth-engine-tree
+)
+
+"${cmd[@]}" --features test-utils --workspace "${excludes[@]}"
+
+# TODO: Slow benchmarks due to too many iterations
+## "${cmd[@]}" -p reth-transaction-pool --features test-utils,arbitrary

.github/workflows/bench.yml

@@ -1,4 +1,4 @@
-# Runs benchmarks on serialization/deserialization of storage values and keys.
+# Runs benchmarks.
 
 on:
   pull_request:
@@ -9,55 +9,33 @@ on:
 env:
   CARGO_TERM_COLOR: always
   BASELINE: base
-  IAI_CALLGRIND_RUNNER: iai-callgrind-runner
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 name: bench
 jobs:
-  iai:
+  codspeed:
     runs-on:
       group: Reth
-    # Only run benchmarks in merge groups and on main
-    if: github.event_name != 'pull_request'
     steps:
       - uses: actions/checkout@v4
-      - name: Install Valgrind
-        run: sudo apt update && sudo apt install valgrind
+        with:
+          submodules: true
+      - uses: rui314/setup-mold@v1
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
         with:
           cache-on-failure: true
-      - name: Install cargo-binstall
-        uses: taiki-e/install-action@cargo-binstall
-      - name: Install iai-callgrind-runner
-        run: |
-          echo "::group::Install"
-          version=$(cargo metadata --format-version=1 |\
-            jq '.packages[] | select(.name == "iai-callgrind").version' |\
-            tr -d '"'
-          )
-          cargo binstall iai-callgrind-runner --version $version --no-confirm --no-symlinks --force
-          echo "::endgroup::"
-          echo "::group::Verification"
-          which iai-callgrind-runner
-          echo "::endgroup::"
-      - name: Checkout base
-        uses: actions/checkout@v4
+      - name: Install cargo-codspeed
+        uses: taiki-e/install-action@v2
         with:
-          ref: ${{ github.base_ref || 'main' }}
-      # On `main` branch, generates test vectors and serializes them to disk using `serde-json`.
-      - name: Generate test vectors
-        run: cargo run --bin reth --features dev -- test-vectors tables
-      # Runs iai and stores `main` baseline report for comparison later on $BASELINE.
-      - name: Save baseline
-        run: cargo bench -p reth-db --bench iai --profile profiling --features test-utils -- --save-baseline=$BASELINE
-      - name: Checkout PR
-        uses: actions/checkout@v4
+          tool: cargo-codspeed
+      - name: Build the benchmark target(s)
+        run: ./.github/scripts/codspeed-build.sh
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v3
         with:
-          clean: false
-      # Runs iai on incoming merge using previously generated test-vectors and compares the report against `main` report.
-      - name: Compare PR benchmarks
-        run: cargo bench -p reth-db --bench iai --profile profiling --features test-utils -- --baseline=$BASELINE
+          run: cargo codspeed run --workspace
+          token: ${{ secrets.CODSPEED_TOKEN }}

.gitignore

@@ -19,7 +19,7 @@ target/
 testdata/micro/db
 
 # Generated data for stage benchmarks
-crates/stages/testdata
+crates/stages/stages/testdata
 
 # Prometheus data dir
 data/

Cargo.lock

@@ -601,8 +601,7 @@ toml = "0.8"
 # misc-testing
 arbitrary = "1.3"
 assert_matches = "1.5.0"
-criterion = "0.5"
-iai-callgrind = "0.14"
+criterion = { package = "codspeed-criterion-compat", version = "2.7" }
 pprof = "0.14"
 proptest = "1.4"
 proptest-derive = "0.5"

Cargo.toml

@@ -77,7 +77,8 @@ test-utils = [
 	"reth-db-api/test-utils",
 	"reth-provider/test-utils",
 	"reth-trie-db/test-utils",
-	"reth-trie/test-utils"
+	"reth-trie/test-utils",
+	"reth-trie-parallel/test-utils"
 ]
 optimism = [
 	"reth-primitives/optimism",

crates/blockchain-tree/Cargo.toml

@@ -120,6 +120,8 @@ test-utils = [
     "reth-static-file",
     "reth-tracing",
     "reth-trie/test-utils",
-    "reth-trie-db/test-utils",
+    "reth-trie-sparse/test-utils",
     "reth-prune-types?/test-utils",
+    "reth-trie-db/test-utils",
+    "reth-trie-parallel/test-utils",
 ]

crates/engine/tree/Cargo.toml

@@ -7,7 +7,7 @@ use revm_primitives::{
     Account, AccountInfo, AccountStatus, Address, EvmState, EvmStorage, EvmStorageSlot, HashMap,
     B256, U256,
 };
-use std::thread;
+use std::{hint::black_box, thread};
 
 /// Creates a mock state with the specified number of accounts for benchmarking
 fn create_bench_state(num_accounts: usize) -> EvmState {
@@ -47,7 +47,7 @@ impl StdStateRootTask {
 
     fn run(self) {
         while let Ok(state) = self.rx.recv() {
-            criterion::black_box(state);
+            black_box(state);
         }
     }
 }
@@ -64,7 +64,7 @@ impl CrossbeamStateRootTask {
 
     fn run(self) {
         while let Ok(state) = self.rx.recv() {
-            criterion::black_box(state);
+            black_box(state);
         }
     }
 }

crates/engine/tree/benches/channel_perf.rs

@@ -3,7 +3,7 @@
 
 #![allow(missing_docs)]
 
-use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
 use reth_engine_tree::tree::root::{StateRootConfig, StateRootTask};
 use reth_evm::system_calls::OnStateHook;
 use reth_primitives::{Account as RethAccount, StorageEntry};
@@ -22,6 +22,7 @@ use revm_primitives::{
     Account as RevmAccount, AccountInfo, AccountStatus, Address, EvmState, EvmStorageSlot, HashMap,
     B256, KECCAK_EMPTY, U256,
 };
+use std::hint::black_box;
 
 #[derive(Debug, Clone)]
 struct BenchParams {

crates/engine/tree/benches/state_root_task.rs

@@ -975,6 +975,5 @@ mod tests {
                 &mut dest,
             );
         }
-        std::hint::black_box(());
     }
 }

crates/net/ecies/src/algorithm.rs

@@ -69,12 +69,10 @@ fn validate_blob_tx(
 
     // for now we just use the default SubPoolLimit
     group.bench_function(group_id, |b| {
+        let kzg_settings = kzg_settings.get();
         b.iter_with_setup(setup, |(tx, blob_sidecar)| {
-            if let Err(err) =
-                std::hint::black_box(tx.validate_blob(&blob_sidecar, kzg_settings.get()))
-            {
-                println!("Validation failed: {err:?}");
-            }
+            let r = tx.validate_blob(&blob_sidecar, kzg_settings);
+            (r, tx, blob_sidecar)
         });
     });
 }