Make benchmarks only run with release builds

.github/workflows/Benchmarks.yml

@@ -57,7 +57,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run Benchmarks
-        run: just bench-ci main release ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
+        run: just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
 
       - uses: actions/upload-artifact@v4
         with:

.github/workflows/dep_rust.yml

@@ -151,5 +151,5 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
+          just bench-ci main ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
         if: ${{ matrix.config == 'release' }}

Justfile

@@ -229,11 +229,13 @@ bench-download os hypervisor cpu tag="":
     tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
-bench-ci baseline target=default-target features="":
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+# Benchmarks only run with release builds for performance consistency
+bench-ci baseline features="":
+    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
 
-bench target=default-target features="":
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+# Benchmarks only run with release builds for performance consistency  
+bench features="":
+    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
 
 ###############
 ### FUZZING ###

docs/benchmarking-hyperlight.md

@@ -72,4 +72,4 @@ Found 1 outliers among 100 measurements (1.00%)
 
 ## Running benchmarks locally
 
-Use `just bench [debug/release]` parameter to run benchmarks. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.
+Use `just bench` to run benchmarks. Benchmarks only run with release builds for performance consistency. Comparing local benchmarks results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.

src/hyperlight_host/benches/benchmarks.rs

@@ -14,6 +14,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Benchmarks are only meaningful and should only run with optimized builds.
+// Unoptimized builds have different performance characteristics and would not provide
+// useful benchmarking data for performance regression testing.
+#[cfg(unoptimized_build)]
+compile_error!(
+    "Benchmarks must be run with optimized builds only. Use `cargo bench --release` or `just bench`."
+);
+
 use criterion::{Criterion, criterion_group, criterion_main};
 use hyperlight_host::GuestBinary;
 use hyperlight_host::sandbox::{

src/hyperlight_host/build.rs

@@ -85,6 +85,24 @@ fn main() -> Result<()> {
         );
     }
 
+    // Set a cfg flag based on optimization level for benchmarks
+    // Benchmarks should only run with optimized builds (opt-level 1+)
+    println!("cargo:rustc-check-cfg=cfg(unoptimized_build)");
+    println!("cargo:rustc-check-cfg=cfg(optimized_build)");
+
+    if let Ok(opt_level) = std::env::var("OPT_LEVEL") {
+        if opt_level == "0" {
+            // Unoptimized build - benchmarks should not run
+            println!("cargo:rustc-cfg=unoptimized_build");
+        } else {
+            // Optimized build - benchmarks can run
+            println!("cargo:rustc-cfg=optimized_build");
+        }
+    } else {
+        // Fallback: if we can't determine opt level, assume unoptimized to be safe
+        println!("cargo:rustc-cfg=unoptimized_build");
+    }
+
     // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))]
     // and #[cfg(mshv)] == #[cfg(all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux"))].
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].