From d14007343dd4baa340cf06806dfdb9a910ae4120 Mon Sep 17 00:00:00 2001 From: Jorge Prendes Date: Fri, 12 Jun 2026 09:42:53 +0100 Subject: [PATCH 1/3] Add hyperlight-ci crate for benchmark orchestration and reporting Introduce a new internal tooling crate (hyperlight-ci) that provides: - bench subcommand: Runs criterion benchmarks in parallel via criterion-swarm. Features include: - Configurable parallelism (-j N, defaults to all P-cores) - Configurable output modes (spinner, stream, summary) - Support for pre-built binaries (--binary) to skip rebuilds - Trailing args forwarded to criterion (filter, --exact, etc.) - bench-report subcommand: Generates markdown comparison tables from criterion's target/criterion/ JSON output via criterion-markdown. Features include: - Benchmark discovery via criterion-swarm - Optional allowlist filtering via --binary or trailing args - Output to stdout This replaces ad-hoc benchmark scripting with a unified tool suitable for both local development and CI report generation. Signed-off-by: Jorge Prendes --- Cargo.lock | 281 ++++++++++++++++++++++---- Cargo.toml | 1 + src/hyperlight_ci/Cargo.toml | 19 ++ src/hyperlight_ci/src/bench.rs | 138 +++++++++++++ src/hyperlight_ci/src/bench_report.rs | 87 ++++++++ src/hyperlight_ci/src/main.rs | 46 +++++ 6 files changed, 532 insertions(+), 40 deletions(-) create mode 100644 src/hyperlight_ci/Cargo.toml create mode 100644 src/hyperlight_ci/src/bench.rs create mode 100644 src/hyperlight_ci/src/bench_report.rs create mode 100644 src/hyperlight_ci/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index ba73df16d..463cfb1d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,18 +58,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] -name = "anstream" -version = "0.6.21" +name = "ansi-replace" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "7f8b155ab93213f41c886d3a46e335258428e52c7cf868e25cf099d50274496d" dependencies = [ - "anstyle", - "anstyle-parse 0.2.7", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", + "regex", + "stable-pattern", ] [[package]] @@ -79,7 +74,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", - "anstyle-parse 1.0.0", + "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", @@ -93,15 +88,6 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - [[package]] name = "anstyle-parse" version = "1.0.0" @@ -117,7 +103,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -128,7 +114,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -499,25 +485,38 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.58" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63be97961acde393029492ce0be7a1af7e323e6bae9511ebfac33751be5e6806" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] name = "clap_builder" -version = "4.5.58" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f13174bda5dfd69d7e947827e5af4b0f2f94a4a3ee92912fba07a66150f21e2" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ - "anstream 0.6.21", + "anstream", "anstyle", "clap_lex", "strsim", ] +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "1.0.0" @@ -530,6 +529,19 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -609,6 +621,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cpu-pin" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb5bc1be026f7f066429ce0611e23a341db91b91ed701de4a1432d01d3ed1105" +dependencies = [ + "libc", + "mach2 0.6.0", + "once_cell", + "tokio", + "windows", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -661,6 +686,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "criterion-markdown" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "742526c4f51cc1d5f812d4270135bada2e951d9a38c0d04d907bfb7b63da70d1" +dependencies = [ + "anyhow", + "serde", + "serde_json", +] + [[package]] name = "criterion-plot" version = "0.8.2" @@ -671,6 +707,23 @@ dependencies = [ "itertools 0.13.0", ] +[[package]] +name = "criterion-swarm" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82693ab3e80479ca52770515f3837028800e365aac48a83edb86f920a6783232" +dependencies = [ + "ansi-replace", + "anyhow", + "clap", + "cpu-pin", + "indicatif", + "regex", + "serde_json", + "simple-pool", + "tokio", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -769,7 +822,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -815,6 +868,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "endian-type" version = "0.1.2" @@ -837,7 +896,7 @@ version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ - "anstream 1.0.0", + "anstream", "anstyle", "env_filter", "jiff", @@ -857,7 +916,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1190,7 +1249,7 @@ dependencies = [ "gobject-sys", "libc", "system-deps", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1456,6 +1515,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlight-ci" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "criterion-markdown", + "criterion-swarm", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "hyperlight-common" version = "0.15.0" @@ -1626,7 +1698,7 @@ dependencies = [ "vmm-sys-util", "windows", "windows-result", - "windows-sys", + "windows-sys 0.61.2", "windows-version", ] @@ -1820,6 +1892,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2105,6 +2190,12 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + [[package]] name = "macho-unwind-info" version = "0.5.0" @@ -2216,7 +2307,7 @@ checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2280,7 +2371,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2313,6 +2404,12 @@ dependencies = [ "syn", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.39.0" @@ -2324,6 +2421,12 @@ dependencies = [ "ruzstd", ] +[[package]] +name = "object-id" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c587bd1cd63959a8520442afc0f92a875d83deea175c7b48dd9f104a2c5070a9" + [[package]] name = "once_cell" version = "1.21.4" @@ -2785,7 +2888,7 @@ dependencies = [ "bindgen 0.70.1", "libc", "libproc", - "mach2", + "mach2 0.4.3", "winapi", ] @@ -3152,7 +3255,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3363,6 +3466,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simple-pool" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "073382259dbeb56c3eaab04a1d330459f6490d1e518b2a8ee441c8bd00dbc092" +dependencies = [ + "object-id", + "parking_lot", +] + [[package]] name = "sketches-ddsketch" version = "0.3.0" @@ -3388,7 +3501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3406,6 +3519,15 @@ dependencies = [ "lock_api", ] +[[package]] +name = "stable-pattern" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4564168c00635f88eaed410d5efa8131afa8d8699a612c80c455a0ba05c21045" +dependencies = [ + "memchr", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3478,7 +3600,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3553,7 +3675,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3913,6 +4035,12 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -4195,7 +4323,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4305,6 +4433,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -4314,6 +4451,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows-threading" version = "0.2.1" @@ -4332,6 +4485,54 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.14" diff --git a/Cargo.toml b/Cargo.toml index e9b69f40d..6650dcbf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ default-members = [ "src/hyperlight_testing", ] members = [ + "src/hyperlight_ci", "src/hyperlight_common", "src/hyperlight_guest", "src/hyperlight_host", diff --git a/src/hyperlight_ci/Cargo.toml b/src/hyperlight_ci/Cargo.toml new file mode 100644 index 000000000..11adb4d2c --- /dev/null +++ b/src/hyperlight_ci/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "hyperlight-ci" +edition = "2021" +# fields intentionally not set, to avoid accidentally publishing this crate to crates.io +description = """ +Hyperlight's CI and development tools. +""" + +[lints] +workspace = true + +[dependencies] +anyhow = "1" +clap = { version = "4.6.1", features = ["derive"] } +criterion-markdown = "0.1.1" +criterion-swarm = "0.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1", features = ["rt"] } \ No newline at end of file diff --git a/src/hyperlight_ci/src/bench.rs b/src/hyperlight_ci/src/bench.rs new file mode 100644 index 000000000..8b8904f6d --- /dev/null +++ b/src/hyperlight_ci/src/bench.rs @@ -0,0 +1,138 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +//! The `bench` subcommand: runs criterion benchmarks in parallel via criterion-swarm. + +use std::io::IsTerminal; +use std::path::PathBuf; + +use anyhow::Context; +use criterion_swarm::{CriterionSwarm, OutputMode}; + +/// An output mode flag for `--build-output` / `--benchmarks-output`. +#[derive(Clone, Debug)] +pub(crate) struct OutputModeFlags(OutputMode); + +impl OutputModeFlags { + /// Parse a single token into an `OutputMode` flag. + fn parse_one(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "spinner" => Ok(OutputMode::SPINNER), + "stream" => Ok(OutputMode::STREAM), + "summary" => Ok(OutputMode::SUMMARY), + "none" | "silent" => Ok(OutputMode::SILENT), + other => Err(format!( + "unknown output mode `{other}` (expected: spinner, stream, summary, none)" + )), + } + } +} + +impl std::str::FromStr for OutputModeFlags { + type Err = String; + fn from_str(s: &str) -> Result { + let mut mode = OutputMode::SILENT; + for part in s.split(',') { + mode = mode | Self::parse_one(part)?; + } + Ok(Self(mode)) + } +} + +/// Merge a `Vec` into a single `OutputMode` by OR-ing them together. +fn merge_output_modes(flags: &[OutputModeFlags]) -> OutputMode { + flags.iter().fold(OutputMode::SILENT, |acc, f| acc | f.0) +} + +/// Command-line arguments for the `bench` subcommand. +#[derive(clap::Args)] +pub struct BenchArgs { + /// Pre-built benchmark binary to use (skip build step; can be specified multiple times) + #[arg(long)] + pub binary: Vec, + + /// Number of benchmarks to run in parallel (0 = all P-cores, default: 0) + #[arg(long, short, default_value_t = 0)] + pub jobs: usize, + + /// Build output mode (comma-separated or repeated): spinner, stream, summary, none + #[arg(long, value_delimiter = ',')] + pub build_output: Vec, + + /// Benchmarks output mode (comma-separated or repeated): spinner, stream, summary, none + #[arg(long, value_delimiter = ',')] + pub benchmarks_output: Vec, + + /// Additional features to pass to cargo when building benchmarks (can be specified multiple times) + #[arg(short = 'F', long)] + pub features: Vec, + + /// Additional arguments to forward to criterion benchmarks + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + pub bench_args: Vec, +} + +pub async fn run(mut args: BenchArgs) -> anyhow::Result<()> { + let mut swarm = CriterionSwarm::builder().jobs(args.jobs); + + if !args.binary.is_empty() { + swarm = swarm.binaries(args.binary); + } + + if !args.features.is_empty() { + swarm = swarm.build_args(["--features".to_string(), args.features.join(",")]); + } + + for arg in args.bench_args { + swarm = swarm.bench_arg(arg); + } + + if args.build_output.is_empty() { + let mode = if std::io::stderr().is_terminal() { + OutputMode::SPINNER | OutputMode::SUMMARY + } else { + OutputMode::STREAM | OutputMode::SUMMARY + }; + args.build_output.push(OutputModeFlags(mode)); + } + + if args.benchmarks_output.is_empty() { + let mode = if std::io::stderr().is_terminal() { + OutputMode::SPINNER | OutputMode::STREAM | OutputMode::SUMMARY + } else { + OutputMode::STREAM | OutputMode::SUMMARY + }; + args.benchmarks_output.push(OutputModeFlags(mode)); + } + + let build_mode = merge_output_modes(&args.build_output); + let bench_mode = merge_output_modes(&args.benchmarks_output); + swarm = swarm.output( + criterion_swarm::ProgressReporter::new() + .build(build_mode) + .benchmarks(bench_mode), + ); + + let swarm = swarm + .prepare() + .await + .context("Failed to prepare criterion swarm")?; + if bench_mode == (bench_mode | OutputMode::SUMMARY) { + let total = swarm.benchmarks().len(); + let jobs = swarm.jobs().min(total); + println!("Running {total} benchmarks with parallelism {jobs}"); + } + swarm.run().await.context("Failed to run criterion swarm") +} diff --git a/src/hyperlight_ci/src/bench_report.rs b/src/hyperlight_ci/src/bench_report.rs new file mode 100644 index 000000000..f733fecec --- /dev/null +++ b/src/hyperlight_ci/src/bench_report.rs @@ -0,0 +1,87 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +//! The `bench-report` subcommand: generates a markdown table from existing +//! criterion benchmark results in `target/criterion/`. + +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::Args; +use criterion_swarm::{CriterionSwarm, NoopReporter}; + +/// Command-line arguments for the `bench-report` subcommand. +#[derive(Args)] +pub struct BenchReportArgs { + /// Benchmark binary to list benchmarks from (can be specified multiple times). + /// When provided, only benchmarks available in these binaries are included. + #[arg(long)] + pub binary: Vec, + + /// Path to the criterion output directory + #[arg(long, default_value = "target/criterion")] + pub criterion_dir: PathBuf, + + /// Wrap the output in a collapsible
tag with the given summary text. + #[arg(long)] + pub collapsible: Option, + + /// Additional arguments to forward to criterion benchmarks (e.g. filter, --exact) + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + pub bench_args: Vec, +} + +/// Entry point for the bench-report subcommand. +pub async fn run(args: BenchReportArgs) -> Result<()> { + let allowlist = build_allowlist(&args).await?; + + let options = criterion_markdown::RenderOptions { + collapsible: args.collapsible, + }; + let markdown = + criterion_markdown::render_with_options(&args.criterion_dir, &allowlist, &options)?; + + print!("{markdown}"); + + Ok(()) +} + +/// Builds an allowlist of benchmark full_ids by discovering benchmarks via CriterionSwarm. +/// +/// All trailing arguments (filter, --exact, etc.) are forwarded as bench args +/// to CriterionSwarm so it handles filtering during discovery. +async fn build_allowlist(args: &BenchReportArgs) -> Result> { + let mut swarm = CriterionSwarm::builder(); + + if !args.binary.is_empty() { + swarm = swarm.binaries(&args.binary); + } + + for arg in &args.bench_args { + swarm = swarm.bench_arg(arg); + } + + let discovered = swarm + .output(NoopReporter) + .prepare() + .await + .context("Failed to discover benchmarks")?; + + Ok(discovered + .benchmarks() + .into_iter() + .map(str::to_string) + .collect()) +} diff --git a/src/hyperlight_ci/src/main.rs b/src/hyperlight_ci/src/main.rs new file mode 100644 index 000000000..38e77e85d --- /dev/null +++ b/src/hyperlight_ci/src/main.rs @@ -0,0 +1,46 @@ +/* +Copyright 2025 The Hyperlight Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +mod bench; +mod bench_report; + +use clap::{Parser, Subcommand}; + +#[derive(Parser)] +#[command( + name = "hyperlight-ci", + about = "Hyperlight's CI and development tools" +)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Run benchmarks using the benchmark binary directly + Bench(bench::BenchArgs), + /// Generate a markdown table from existing criterion benchmark results + BenchReport(bench_report::BenchReportArgs), +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + match cli.command { + Commands::Bench(args) => bench::run(args).await, + Commands::BenchReport(args) => bench_report::run(args).await, + } +} From 6abf01f2f0055e4d6e695bcb381edb3adf86f5c3 Mon Sep 17 00:00:00 2001 From: Jorge Prendes Date: Fri, 12 Jun 2026 09:46:50 +0100 Subject: [PATCH 2/3] Integrate hyperlight-ci into CI workflows and Just recipes - Add cargo alias (`cargo ci`) for convenient hyperlight-ci invocation - Update dep_benchmarks workflow to use `cargo ci bench` and generate a markdown report via `cargo ci bench-report`, posting results as a PR comment per hypervisor/cpu matrix entry - Add benchmarks job to ValidatePullRequest workflow with hypervisor and cpu matrix, gated behind docs-only and build-guests checks - Grant pull-requests: write permission for PR comment posting - Simplify Justfile bench recipes to delegate to `cargo ci bench` - Update benchmarking docs to reflect the new workflow Signed-off-by: Jorge Prendes --- .cargo/config.toml | 2 + .github/hyperlight-bot.yml | 8 ++++ .github/workflows/ValidatePullRequest.yml | 56 +++++++++++++++++++++++ .github/workflows/dep_benchmarks.yml | 11 ++++- Justfile | 6 +-- 5 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 .github/hyperlight-bot.yml diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..1831b0a74 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] # command aliases +ci = ["run", "--quiet", "--package=hyperlight-ci", "--"] \ No newline at end of file diff --git a/.github/hyperlight-bot.yml b/.github/hyperlight-bot.yml new file mode 100644 index 000000000..758fc3e55 --- /dev/null +++ b/.github/hyperlight-bot.yml @@ -0,0 +1,8 @@ +# Configuration for the hyperlight-gh-bot GitHub App. +# See: https://github.com/jprendes/hyperlight-gh-bot + +# Name of the artifact containing the comment body. +artifact_name: "pr-comment" + +# Regex matched against the job name to filter which jobs trigger the bot. +job_filter: "post-benchmark-comment" diff --git a/.github/workflows/ValidatePullRequest.yml b/.github/workflows/ValidatePullRequest.yml index 2f6294476..39cdca317 100644 --- a/.github/workflows/ValidatePullRequest.yml +++ b/.github/workflows/ValidatePullRequest.yml @@ -140,6 +140,61 @@ jobs: docs_only: ${{ needs.docs-pr.outputs.docs-only }} secrets: inherit + # Run benchmarks and post results as PR comment + benchmarks: + needs: + - docs-pr + - build-guests + # Required because update-guest-locks is skipped on non-dependabot PRs, + # and a skipped dependency transitively skips all downstream jobs. + # See: https://github.com/actions/runner/issues/2205 + if: ${{ !cancelled() && !failure() }} + strategy: + fail-fast: false + matrix: + hypervisor: ['hyperv-ws2025', mshv3, kvm] + cpu: [amd, intel] + uses: ./.github/workflows/dep_benchmarks.yml + secrets: inherit + with: + docs_only: ${{ needs.docs-pr.outputs.docs-only }} + hypervisor: ${{ matrix.hypervisor }} + cpu: ${{ matrix.cpu }} + + # Combine benchmark reports into a single artifact for the hyperlight-gh-bot + # to post as a PR comment. Only runs for PRs (not merge groups) with code changes. + benchmark-comment: + name: post-benchmark-comment + needs: + - docs-pr + - benchmarks + if: ${{ !cancelled() && !failure() && needs.docs-pr.outputs.docs-only == 'false' && github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + steps: + - name: Download benchmark reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: benchmark-report_* + path: reports/ + + - name: Combine benchmark reports + run: | + echo '## Benchmark Results' > pr-comment.md + echo '' >> pr-comment.md + for f in reports/benchmark-report_*/benchmark.md; do + [ -f "$f" ] || continue + cat "$f" >> pr-comment.md + echo '' >> pr-comment.md + done + + - name: Upload PR comment artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pr-comment + path: pr-comment.md + if-no-files-found: warn + retention-days: 1 + spelling: name: spell check with typos runs-on: ubuntu-latest @@ -167,6 +222,7 @@ jobs: - build-test - run-examples - fuzzing + - benchmarks - spelling - license-headers if: always() diff --git a/.github/workflows/dep_benchmarks.yml b/.github/workflows/dep_benchmarks.yml index b0c47be76..5b3b62d06 100644 --- a/.github/workflows/dep_benchmarks.yml +++ b/.github/workflows/dep_benchmarks.yml @@ -56,7 +56,6 @@ on: required: false type: number default: 5 - env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full @@ -135,6 +134,16 @@ jobs: - name: Run benchmarks run: just bench-ci main + - name: Create benchmarks report + run: cargo ci bench-report --collapsible '${{ inputs.hypervisor }} / ${{ inputs.cpu }} (${{ runner.os }})' > target/criterion/benchmark.md + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-report_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }} + path: target/criterion/benchmark.md + if-no-files-found: error + retention-days: 1 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }} diff --git a/Justfile b/Justfile index 401897425..bc8a0ee77 100644 --- a/Justfile +++ b/Justfile @@ -401,12 +401,10 @@ bench-download os hypervisor cpu tag="": # Warning: compares to and then OVERWRITES the given baseline bench-ci baseline features="": - @# Benchmarks are always run with release builds for meaningful results - cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} + cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }} bench features="": - @# Benchmarks are always run with release builds for meaningful results - cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose + cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose ############### ### FUZZING ### From c8864da0d12771816a445e3b0ddba0472818218c Mon Sep 17 00:00:00 2001 From: Jorge Prendes Date: Thu, 18 Jun 2026 17:45:49 +0100 Subject: [PATCH 3/3] Run benchmarks sequentially Signed-off-by: Jorge Prendes --- Justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index bc8a0ee77..fff202169 100644 --- a/Justfile +++ b/Justfile @@ -401,10 +401,10 @@ bench-download os hypervisor cpu tag="": # Warning: compares to and then OVERWRITES the given baseline bench-ci baseline features="": - cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }} + cargo ci bench --jobs=1 {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }} bench features="": - cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose + cargo ci bench --jobs=1 {{ if features == "" {''} else { "--features " + features } }} --verbose ############### ### FUZZING ###