diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 672d6183e..3037c29a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ ---- # Workflow with multiple jobs to test different parts of the project +--- # Workflow with multiple jobs to test different parts of the project name: Continuous Integration @@ -66,7 +66,7 @@ jobs: with: cache-on-failure: true - - name: build # build separately so test logs are actually nice + - name: build # build separately so test logs are actually nice run: cargo build --tests --manifest-path trace_decoder/Cargo.toml - name: test @@ -171,7 +171,7 @@ jobs: uses: actions/checkout@v4 - name: Run the script - run: ./scripts/prove_stdio.sh artifacts/witness_b19807080.json + run: cargo xtask prove-stdio verify artifacts/witness_b19807080.json simple_proof_witness_only: name: Execute bash script to generate the proof witness for a small block. @@ -182,7 +182,7 @@ jobs: uses: actions/checkout@v4 - name: Run the script - run: ./scripts/prove_stdio.sh artifacts/witness_b19807080.json test_only + run: cargo xtask prove-stdio test artifacts/witness_b19807080.json multi_blocks_proof_regular: name: Execute bash script to generate and verify a proof for multiple blocks using parallel proving. @@ -193,4 +193,4 @@ jobs: uses: actions/checkout@v4 - name: Run the script - run: ./scripts/prove_stdio.sh artifacts/witness_b3_b6.json "" use_test_config + run: cargo xtask prove-stdio test artifacts/witness_b3_b6.json --use-test-config diff --git a/scripts/prove_stdio.rs b/scripts/prove_stdio.rs index 40ed979d0..50d15c1a1 100644 --- a/scripts/prove_stdio.rs +++ b/scripts/prove_stdio.rs @@ -27,7 +27,7 @@ pub struct ProveStdioArgs { block_batch_size: u32, /// The directory to output the proof files. If it does not exist, it will /// recursively be created. - #[arg(short = 'o', long, value_hint = ValueHint::DirPath, default_value = ".")] + #[arg(short = 'o', long, value_hint = ValueHint::DirPath, default_value = "./proofs")] output_dir: PathBuf, } @@ -45,51 +45,6 @@ pub fn prove_via_stdio(args: ProveStdioArgs) -> anyhow::Result<()> { match args.mode { RunMode::Test => { - envs.extend([ - ("ARITHMETIC_CIRCUIT_SIZE", "16..18"), - ("BYTE_PACKING_CIRCUIT_SIZE", "8..15"), - ("CPU_CIRCUIT_SIZE", "9..20"), - ("KECCAK_CIRCUIT_SIZE", "7..18"), - ("KECCAK_SPONGE_CIRCUIT_SIZE", "8..14"), - ("LOGIC_CIRCUIT_SIZE", "5..17"), - ("MEMORY_CIRCUIT_SIZE", "17..22"), - ("MEMORY_BEFORE_CIRCUIT_SIZE", "16..20"), - ("MEMORY_AFTER_CIRCUIT_SIZE", "7..20"), - // TODO(Robin): update Poseidon ranges here and below once Kernel ASM supports - ("POSEIDON_CIRCUIT_SIZE", "4..8"), - ]); - let witness_filename = args - .input_witness_file - .to_str() - .ok_or(anyhow::anyhow!("Invalid witness file path"))?; - if witness_filename.contains("witness_b19807080") { - envs.extend([ - ("ARITHMETIC_CIRCUIT_SIZE", "16..18"), - ("BYTE_PACKING_CIRCUIT_SIZE", "8..15"), - ("CPU_CIRCUIT_SIZE", "9..20"), - ("KECCAK_CIRCUIT_SIZE", "7..18"), - ("KECCAK_SPONGE_CIRCUIT_SIZE", "8..14"), - ("LOGIC_CIRCUIT_SIZE", "5..17"), - ("MEMORY_CIRCUIT_SIZE", "17..22"), - ("MEMORY_BEFORE_CIRCUIT_SIZE", "16..20"), - ("MEMORY_AFTER_CIRCUIT_SIZE", "7..20"), - ("POSEIDON_CIRCUIT_SIZE", "4..8"), - ]); - } else if witness_filename.contains("witness_b3_b6") { - envs.extend([ - ("ARITHMETIC_CIRCUIT_SIZE", "16..18"), - ("BYTE_PACKING_CIRCUIT_SIZE", "8..15"), - ("CPU_CIRCUIT_SIZE", "10..20"), - ("KECCAK_CIRCUIT_SIZE", "4..13"), - ("KECCAK_SPONGE_CIRCUIT_SIZE", "8..9"), - ("LOGIC_CIRCUIT_SIZE", "4..14"), - ("MEMORY_CIRCUIT_SIZE", "17..22"), - ("MEMORY_BEFORE_CIRCUIT_SIZE", "16..18"), - ("MEMORY_AFTER_CIRCUIT_SIZE", "7..8"), - ("POSEIDON_CIRCUIT_SIZE", "4..8"), - ]); - } - let mut cmd = prove_command(args, envs)?; let status = cmd.spawn()?.wait()?; ensure!(status.success(), "command failed with {}", status); @@ -105,7 +60,9 @@ pub fn prove_via_stdio(args: ProveStdioArgs) -> anyhow::Result<()> { ensure!(status.success(), "command failed with {}", status); // Construct the command to run. + add_verify_envs(&args, &mut envs)?; let mut cmd = prove_command(args, envs)?; + // Time the proving. let start = std::time::Instant::now(); let status = cmd.spawn()?.wait()?; @@ -117,6 +74,55 @@ pub fn prove_via_stdio(args: ProveStdioArgs) -> anyhow::Result<()> { } } +fn add_verify_envs(args: &ProveStdioArgs, envs: &mut Vec<(&str, &str)>) -> anyhow::Result<()> { + let witness_filename = args + .input_witness_file + .to_str() + .ok_or(anyhow::anyhow!("Invalid witness file path"))?; + if witness_filename.contains("witness_b19807080") { + envs.extend([ + ("ARITHMETIC_CIRCUIT_SIZE", "16..18"), + ("BYTE_PACKING_CIRCUIT_SIZE", "8..15"), + ("CPU_CIRCUIT_SIZE", "9..20"), + ("KECCAK_CIRCUIT_SIZE", "7..18"), + ("KECCAK_SPONGE_CIRCUIT_SIZE", "8..14"), + ("LOGIC_CIRCUIT_SIZE", "5..17"), + ("MEMORY_CIRCUIT_SIZE", "17..22"), + ("MEMORY_BEFORE_CIRCUIT_SIZE", "16..20"), + ("MEMORY_AFTER_CIRCUIT_SIZE", "7..20"), + ("POSEIDON_CIRCUIT_SIZE", "4..8"), + ]); + } else if witness_filename.contains("witness_b3_b6") { + envs.extend([ + ("ARITHMETIC_CIRCUIT_SIZE", "16..18"), + ("BYTE_PACKING_CIRCUIT_SIZE", "8..15"), + ("CPU_CIRCUIT_SIZE", "10..20"), + ("KECCAK_CIRCUIT_SIZE", "4..13"), + ("KECCAK_SPONGE_CIRCUIT_SIZE", "8..9"), + ("LOGIC_CIRCUIT_SIZE", "4..14"), + ("MEMORY_CIRCUIT_SIZE", "17..22"), + ("MEMORY_BEFORE_CIRCUIT_SIZE", "16..18"), + ("MEMORY_AFTER_CIRCUIT_SIZE", "7..8"), + ("POSEIDON_CIRCUIT_SIZE", "4..8"), + ]); + } else { + envs.extend([ + ("ARITHMETIC_CIRCUIT_SIZE", "16..18"), + ("BYTE_PACKING_CIRCUIT_SIZE", "8..15"), + ("CPU_CIRCUIT_SIZE", "9..20"), + ("KECCAK_CIRCUIT_SIZE", "7..18"), + ("KECCAK_SPONGE_CIRCUIT_SIZE", "8..14"), + ("LOGIC_CIRCUIT_SIZE", "5..17"), + ("MEMORY_CIRCUIT_SIZE", "17..22"), + ("MEMORY_BEFORE_CIRCUIT_SIZE", "16..20"), + ("MEMORY_AFTER_CIRCUIT_SIZE", "7..20"), + // TODO(Robin): update Poseidon ranges here and below once Kernel ASM supports + ("POSEIDON_CIRCUIT_SIZE", "4..8"), + ]); + } + Ok(()) +} + fn prove_command(args: ProveStdioArgs, envs: Vec<(&str, &str)>) -> anyhow::Result { let witness_file = File::open(&args.input_witness_file)?; let mut cmd = Command::new("cargo"); @@ -139,10 +145,10 @@ fn prove_command(args: ProveStdioArgs, envs: Vec<(&str, &str)>) -> anyhow::Resul args.output_dir .to_str() .ok_or(anyhow::anyhow!("Invalid output dir path"))?, - "stdio", ]); if args.use_test_config { cmd.arg("--use-test-config"); } + cmd.arg("stdio"); Ok(cmd) } diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh deleted file mode 100755 index 04bbbbe6b..000000000 --- a/scripts/prove_stdio.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash -# ------------------------------------------------------------------------------ -set -exo pipefail - -# Run prover with the parsed input from the standard terminal. -# To generate the json input file, use the `rpc` tool, for example: -# `cargo run --bin rpc -- fetch --rpc-url http://127.0.0.1:8546 --start-block 2 --end-block 5 > witness.json` - -# Args: -# 1 --> Input witness json file -# 2 --> Test run only flag `test_only` (optional) - -# We're going to set the parallelism in line with the total cpu count -if [[ "$OSTYPE" == "darwin"* ]]; then - num_procs=$(sysctl -n hw.physicalcpu) -else - num_procs=$(nproc) -fi - -# Force the working directory to always be the `tools/` directory. -REPO_ROOT=$(git rev-parse --show-toplevel) -PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" - -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" -echo "Block batch size: $BLOCK_BATCH_SIZE" - -OUTPUT_LOG="${REPO_ROOT}/output.log" -PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" -TEST_OUT_PATH="${REPO_ROOT}/test.out" - -# Configured Rayon and Tokio with rough defaults -export RAYON_NUM_THREADS=$num_procs -export TOKIO_WORKER_THREADS=$num_procs - -export RUST_MIN_STACK=33554432 -export RUST_BACKTRACE=full -export RUST_LOG=info -# Script users are running locally, and might benefit from extra perf. -# See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' - -INPUT_FILE=$1 -TEST_ONLY=$2 -USE_TEST_CONFIG=$3 - -if [[ $INPUT_FILE == "" ]]; then - echo "Please provide witness json input file, e.g. artifacts/witness_b19240705.json" - exit 1 -fi - -# Circuit sizes only matter in non test_only mode. -if ! [[ $TEST_ONLY == "test_only" ]]; then - if [[ $INPUT_FILE == *"witness_b19807080"* ]]; then - # These sizes are configured specifically for block 19807080. Don't use this in other scenarios - echo "Using specific circuit sizes for witness_b19807080.json" - export ARITHMETIC_CIRCUIT_SIZE="16..18" - export BYTE_PACKING_CIRCUIT_SIZE="8..15" - export CPU_CIRCUIT_SIZE="9..20" - export KECCAK_CIRCUIT_SIZE="7..18" - export KECCAK_SPONGE_CIRCUIT_SIZE="8..14" - export LOGIC_CIRCUIT_SIZE="5..17" - export MEMORY_CIRCUIT_SIZE="17..22" - export MEMORY_BEFORE_CIRCUIT_SIZE="16..20" - export MEMORY_AFTER_CIRCUIT_SIZE="7..20" - # TODO(Robin): update Poseidon ranges here and below once Kernel ASM supports Poseidon ops - export POSEIDON_CIRCUIT_SIZE="4..8" - elif [[ $INPUT_FILE == *"witness_b3_b6"* ]]; then - # These sizes are configured specifically for custom blocks 3 to 6. Don't use this in other scenarios - echo "Using specific circuit sizes for witness_b3_b6.json" - export ARITHMETIC_CIRCUIT_SIZE="16..18" - export BYTE_PACKING_CIRCUIT_SIZE="8..15" - export CPU_CIRCUIT_SIZE="10..20" - export KECCAK_CIRCUIT_SIZE="4..13" - export KECCAK_SPONGE_CIRCUIT_SIZE="8..9" - export LOGIC_CIRCUIT_SIZE="4..14" - export MEMORY_CIRCUIT_SIZE="17..22" - export MEMORY_BEFORE_CIRCUIT_SIZE="16..18" - export MEMORY_AFTER_CIRCUIT_SIZE="7..8" - export POSEIDON_CIRCUIT_SIZE="4..8" - else - export ARITHMETIC_CIRCUIT_SIZE="16..21" - export BYTE_PACKING_CIRCUIT_SIZE="8..21" - export CPU_CIRCUIT_SIZE="8..21" - export KECCAK_CIRCUIT_SIZE="4..20" - export KECCAK_SPONGE_CIRCUIT_SIZE="8..17" - export LOGIC_CIRCUIT_SIZE="4..21" - export MEMORY_CIRCUIT_SIZE="17..24" - export MEMORY_BEFORE_CIRCUIT_SIZE="16..23" - export MEMORY_AFTER_CIRCUIT_SIZE="7..23" - export POSEIDON_CIRCUIT_SIZE="4..8" - fi -fi - - -# If we run ./prove_stdio.sh test_only, we'll generate a dummy -# proof. This is useful for quickly testing decoding and all of the -# other non-proving code. -if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --quiet --release --package zero --bin leader -- \ - --test-only \ - --runtime in-memory \ - --load-strategy on-demand \ - --block-batch-size "$BLOCK_BATCH_SIZE" \ - --proof-output-dir "$PROOF_OUTPUT_DIR" \ - stdio < "$INPUT_FILE" &> "$TEST_OUT_PATH" - - if grep -q 'All proof witnesses have been generated successfully.' "$TEST_OUT_PATH"; then - echo -e "\n\nSuccess - Note this was just a test, not a proof" - rm "$TEST_OUT_PATH" - exit - else - # Some error occurred, display the logs and exit. - cat "$TEST_OUT_PATH" - echo "Failed to create proof witnesses. See $TEST_OUT_PATH for more details." - exit 1 - fi -fi - -cargo build --release --jobs "$num_procs" - -start_time=$(date +%s%N) - -cmd=("${REPO_ROOT}/target/release/leader" --runtime in-memory \ - --load-strategy on-demand \ - --block-batch-size "$BLOCK_BATCH_SIZE") - -if [[ "$USE_TEST_CONFIG" == "use_test_config" ]]; then - cmd+=("--use-test-config") -fi - -"${cmd[@]}" --proof-output-dir "$PROOF_OUTPUT_DIR" stdio < "$INPUT_FILE" &> "$OUTPUT_LOG" -end_time=$(date +%s%N) - -grep "Successfully wrote to disk proof file " "$OUTPUT_LOG" | awk '{print $NF}' | tee "$PROOFS_FILE_LIST" -if [ ! -s "$PROOFS_FILE_LIST" ]; then - # Some error occurred, display the logs and exit. - cat "$OUTPUT_LOG" - echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG" - exit 1 -fi - -while read -r proof_file; -do - echo "Verifying proof file $proof_file" - verify_file=$PROOF_OUTPUT_DIR/verify_$(basename "$proof_file").out - "${REPO_ROOT}/target/release/verifier" -f "$proof_file" | tee "$verify_file" - if grep -q 'All proofs verified successfully!' "$verify_file"; then - echo "Proof verification for file $proof_file successful"; - rm "$verify_file" # we keep the generated proof for potential reuse - else - # Some error occurred with verification, display the logs and exit. - cat "$verify_file" - echo "There was an issue with proof verification. See $verify_file for more details."; - exit 1 - fi -done < "$PROOFS_FILE_LIST" - -duration_ns=$((end_time - start_time)) -duration_sec=$(echo "$duration_ns / 1000000000" | bc -l) - -echo "Success!" -echo "Proving duration: $duration_sec seconds" -echo "Note, this duration is inclusive of circuit handling and overall process initialization"; - -# Clean up in case of success -rm "$OUTPUT_LOG" -