diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index 64df29bb2..b0749e881 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -57,7 +57,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Benchmarks - run: just bench-ci main release ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + run: just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 389f24faa..f14b1cf66 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -151,5 +151,5 @@ jobs: - name: Run benchmarks run: | - just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} + just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} if: ${{ matrix.config == 'release' }} diff --git a/Justfile b/Justfile index 66b06a587..c9482aaa8 100644 --- a/Justfile +++ b/Justfile @@ -229,11 +229,13 @@ bench-download os hypervisor cpu tag="": tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1 # Warning: compares to and then OVERWRITES the given baseline -bench-ci baseline target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} +# Benchmarks only run with release builds for performance consistency +bench-ci baseline features="": + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} -bench target=default-target features="": - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose +# Benchmarks only run with release builds for performance consistency +bench features="": + cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose ############### ### FUZZING ### diff --git a/docs/benchmarking-hyperlight.md b/docs/benchmarking-hyperlight.md index 3ae56d55c..399a32fc1 100644 --- a/docs/benchmarking-hyperlight.md +++ b/docs/benchmarking-hyperlight.md @@ -72,4 +72,4 @@ Found 1 outliers among 100 measurements (1.00%) ## Running benchmarks locally -Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. +Use `just bench` to run benchmarks. Benchmarks only run with release builds for performance consistency. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub. diff --git a/src/hyperlight_host/benches/benchmarks.rs b/src/hyperlight_host/benches/benchmarks.rs index a0943a1ea..5143a0f33 100644 --- a/src/hyperlight_host/benches/benchmarks.rs +++ b/src/hyperlight_host/benches/benchmarks.rs @@ -14,6 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Benchmarks are only meaningful and should only run with optimized builds. +// Unoptimized builds have different performance characteristics and would not provide +// useful benchmarking data for performance regression testing. +#[cfg(unoptimized_build)] +compile_error!( + "Benchmarks must be run with optimized builds only. Use `cargo bench --release` or `just bench`." +); + use criterion::{Criterion, criterion_group, criterion_main}; use hyperlight_host::GuestBinary; use hyperlight_host::sandbox::{ diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 75f9eba53..287645efa 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -85,6 +85,24 @@ fn main() -> Result<()> { ); } + // Set a cfg flag based on optimization level for benchmarks + // Benchmarks should only run with optimized builds (opt-level 1+) + println!("cargo:rustc-check-cfg=cfg(unoptimized_build)"); + println!("cargo:rustc-check-cfg=cfg(optimized_build)"); + + if let Ok(opt_level) = std::env::var("OPT_LEVEL") { + if opt_level == "0" { + // Unoptimized build - benchmarks should not run + println!("cargo:rustc-cfg=unoptimized_build"); + } else { + // Optimized build - benchmarks can run + println!("cargo:rustc-cfg=optimized_build"); + } + } else { + // Fallback: if we can't determine opt level, assume unoptimized to be safe + println!("cargo:rustc-cfg=unoptimized_build"); + } + // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))] // and #[cfg(mshv)] == #[cfg(all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux"))]. // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].