diff --git a/AGENTS.md b/AGENTS.md index a1af7e6826..53203b9cb6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ You are responsible for the full lifecycle of a task: understanding intent, plan 2. **Plan** - Make a short plan for non-trivial changes; keep scope tight to the user request. - - Identify affected workspace members (`crates/*`, `xtask`, `ffi`, `benches`, `fuzz`, `web-client`) and API boundaries. + - Identify affected workspace members (`crates/*`, `xtask`, `ffi`, `fuzz`, `web-client`) and API boundaries. - Prefer root-cause fixes over local workarounds. 3. **Documentation** @@ -76,7 +76,8 @@ See [skills.sh](https://skills.sh) for more on the `npx skills` command. - **`fuzz/`**: Fuzz targets/corpus for robustness testing. - **`ffi/`**: Native library + .NET bindings and examples. - **`web-client/`**: Browser/web-component/Svelte client artifacts. -- **`benches/`**: Benchmarks and perf-related code. +- **`crates/ironrdp-bench/`**: Consolidated benchmark package with Criterion targets, shared workloads, and standalone CLI binaries. + Crate-local benchmarks, such as `crates/ironrdp-bulk/benches/`, stay next to the code they measure. When changing architecture-sensitive crates, preserve tier boundaries and invariants from `ARCHITECTURE.md`. @@ -93,7 +94,6 @@ These crates exist on disk but are not documented in `ARCHITECTURE.md`. Be aware - `ironrdp-egfx` — extended graphics pipeline channel - `ironrdp-rdpdr-native` — native RDPDR backend - `ironrdp-rdpsnd-native` — native RDPSND backend -- `ironrdp-bench` — benchmarking harness - `iron-remote-desktop` (under `crates/`) — remote desktop abstractions ### Workspace Exclusions diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 19a90091ca..9725d3f4b3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -310,6 +310,10 @@ Provides test case generators and oracles for use with fuzzing. Offline direct-TCP RDP capture analysis that routes recovered plaintext and exports payload-free replay diagnostics with rendered framebuffer snapshots. +#### [`crates/ironrdp-bench`](./crates/ironrdp-bench) + +Consolidated benchmark package with Criterion targets, shared workloads, and standalone CLI binaries. + #### [`fuzz`](./fuzz) Fuzz targets for code in core tier. diff --git a/Cargo.lock b/Cargo.lock index d569a538b4..0920382be9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,20 +439,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "benches" -version = "0.0.0" -dependencies = [ - "anyhow", - "async-trait", - "bytesize", - "ironrdp", - "pico-args", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -608,12 +594,6 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "bytesize" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e" - [[package]] name = "calloop" version = "0.13.0" @@ -2688,10 +2668,20 @@ dependencies = [ name = "ironrdp-bench" version = "0.0.0" dependencies = [ + "anyhow", + "async-trait", "criterion", + "ironrdp", + "ironrdp-capture-replay", "ironrdp-graphics", "ironrdp-pdu", "ironrdp-server", + "pico-args", + "sha2 0.10.9", + "tokio", + "toml", + "tracing", + "tracing-subscriber", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1502734461..237c927fed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,6 @@ [workspace] members = [ "crates/*", - "benches", "xtask", "ffi", ] diff --git a/benches/Cargo.toml b/benches/Cargo.toml deleted file mode 100644 index 9d2302bcfe..0000000000 --- a/benches/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "benches" -version = "0.0.0" -description = "IronRDP benchmarks" -publish = false -edition.workspace = true - -[[bin]] -name = "perfenc" -path = "src/perfenc.rs" - -[features] -default = ["qoi", "qoiz"] -qoi = ["ironrdp/qoi"] -qoiz = ["ironrdp/qoiz"] - -[dependencies] -anyhow = "1" -async-trait = "0.1" -bytesize = "2.3" -ironrdp = { path = "../crates/ironrdp", features = [ - "server", - "pdu", - "__bench", -] } -pico-args = "0.5" -tokio = { version = "1", features = ["sync", "fs", "time"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tracing = { version = "0.1", features = ["log"] } - -[lints] -workspace = true diff --git a/benches/src/perfenc.rs b/benches/src/perfenc.rs deleted file mode 100644 index 1e458c688f..0000000000 --- a/benches/src/perfenc.rs +++ /dev/null @@ -1,225 +0,0 @@ -#![allow(unused_crate_dependencies)] // False positives because there are both a library and a binary. -#![allow(clippy::print_stderr)] -#![allow(clippy::print_stdout)] - -use core::num::{NonZeroU16, NonZeroUsize}; -use core::time::Duration; -use std::io::Write as _; -use std::time::Instant; - -use anyhow::Context as _; -use ironrdp::pdu::rdp::capability_sets::{CmdFlags, EntropyBits, LargePointerSupportFlags}; -use ironrdp::server::bench::encoder::{UpdateEncoder, UpdateEncoderCodecs}; -use ironrdp::server::{BitmapUpdate, DesktopSize, DisplayUpdate, PixelFormat, RdpServerDisplayUpdates}; -use tokio::fs::File; -use tokio::io::AsyncReadExt as _; -use tokio::time::sleep; - -#[tokio::main(flavor = "current_thread")] -async fn main() -> Result<(), anyhow::Error> { - setup_logging()?; - let mut args = pico_args::Arguments::from_env(); - - if args.contains(["-h", "--help"]) { - println!("Usage: perfenc [OPTIONS] "); - println!(); - println!("Measure the performance of the IronRDP server encoder, given a raw RGBX video input file."); - println!(); - println!("Options:"); - println!(" --width Width of the display (default: 3840)"); - println!(" --height Height of the display (default: 2400)"); - println!(" --codec Codec to use (default: remotefx)"); - println!(" Valid values: qoi, qoiz, remotefx, bitmap, none"); - println!(" --fps Frames per second (default: none)"); - std::process::exit(0); - } - - let width = args.opt_value_from_str("--width")?.unwrap_or(3840); - let height = args.opt_value_from_str("--height")?.unwrap_or(2400); - let codec = args.opt_value_from_str("--codec")?.unwrap_or_else(OptCodec::default); - let fps = args.opt_value_from_str("--fps")?.unwrap_or(0); - - let filename: String = args.free_from_str().context("missing RGBX input filename")?; - let file = File::open(&filename) - .await - .with_context(|| format!("Failed to open file: {filename}"))?; - - let mut flags = CmdFlags::all(); - let mut update_codecs = UpdateEncoderCodecs::new(); - - match codec { - OptCodec::RemoteFX => update_codecs.set_remotefx(Some((EntropyBits::Rlgr3, 0))), - OptCodec::Bitmap => { - flags -= CmdFlags::SET_SURFACE_BITS; - } - OptCodec::None => {} - #[cfg(feature = "qoi")] - OptCodec::Qoi => update_codecs.set_qoi(Some(0)), - #[cfg(feature = "qoiz")] - OptCodec::QoiZ => update_codecs.set_qoiz(Some(0)), - }; - - // u16::MAX / LargePointerSupportFlags::all(): this benchmark replays a fixed update - // file and isn't exercising the client capability gates, so don't let a - // no-capability default silently start dropping any pointer updates the replay - // file happens to contain. - let mut encoder = UpdateEncoder::new( - DesktopSize { width, height }, - flags, - update_codecs, - 8 * 1024 * 1024, - u16::MAX, - LargePointerSupportFlags::all(), - ) - .context("failed to initialize update encoder")?; - - let mut total_raw = 0u64; - let mut total_enc = 0u64; - let mut n_updates = 0u64; - let mut updates = DisplayUpdates::new(file, DesktopSize { width, height }, fps); - while let Some(up) = updates.next_update().await? { - if let DisplayUpdate::Bitmap(ref up) = up { - total_raw += u64::try_from(up.data.len())?; - } else { - eprintln!("Invalid update"); - break; - } - let mut iter = encoder.update(up); - loop { - let Some(frag) = iter.next().await else { - break; - }; - let len = u64::try_from(frag?.data.len())?; - total_enc += len; - } - n_updates += 1; - print!("."); - std::io::stdout().flush()?; - } - println!(); - - #[expect(clippy::as_conversions, reason = "casting u64 to f64")] - let ratio = total_enc as f64 / total_raw as f64; - let percent = 100.0 - ratio * 100.0; - println!("Encoder: {encoder:?}"); - println!("Nb updates: {n_updates:?}"); - println!( - "Sum of bytes: {}/{} ({:.2}%)", - bytesize::ByteSize(total_enc), - bytesize::ByteSize(total_raw), - percent, - ); - Ok(()) -} - -struct DisplayUpdates { - file: File, - desktop_size: DesktopSize, - fps: u64, - last_update_time: Option, -} - -impl DisplayUpdates { - fn new(file: File, desktop_size: DesktopSize, fps: u64) -> Self { - Self { - file, - desktop_size, - fps, - last_update_time: None, - } - } -} - -#[async_trait::async_trait] -impl RdpServerDisplayUpdates for DisplayUpdates { - async fn next_update(&mut self) -> ironrdp::server::ServerResult> { - use ironrdp::server::ServerErrorExt as _; - - let stride = self.desktop_size.width as usize * 4; - let frame_size = stride * self.desktop_size.height as usize; - let mut buf = vec![0u8; frame_size]; - // FIXME: AsyncReadExt::read_exact is not cancellation safe. - self.file - .read_exact(&mut buf) - .await - .map_err(|e| ironrdp::server::ServerError::io("read exact", e))?; - - let now = Instant::now(); - if let Some(last_update_time) = self.last_update_time { - let elapsed = now - last_update_time; - if self.fps > 0 && elapsed < Duration::from_millis(1000 / self.fps) { - sleep(Duration::from_millis( - 1000 / self.fps - - u64::try_from(elapsed.as_millis()) - .map_err(|e| ironrdp::server::ServerError::custom("invalid `elapsed millis`", e))?, - )) - .await; - } - } - self.last_update_time = Some(now); - - let up = DisplayUpdate::Bitmap(BitmapUpdate { - x: 0, - y: 0, - width: NonZeroU16::new(self.desktop_size.width) - .ok_or_else(|| ironrdp::server::ServerError::reason("perfenc", "width cannot be zero"))?, - height: NonZeroU16::new(self.desktop_size.height) - .ok_or_else(|| ironrdp::server::ServerError::reason("perfenc", "height cannot be zero"))?, - format: PixelFormat::RgbX32, - data: buf.into(), - stride: NonZeroUsize::new(stride) - .ok_or_else(|| ironrdp::server::ServerError::reason("perfenc", "stride cannot be zero"))?, - }); - Ok(Some(up)) - } -} - -fn setup_logging() -> anyhow::Result<()> { - use tracing::metadata::LevelFilter; - use tracing_subscriber::EnvFilter; - use tracing_subscriber::prelude::*; - - let fmt_layer = tracing_subscriber::fmt::layer().compact(); - - let env_filter = EnvFilter::builder() - .with_default_directive(LevelFilter::WARN.into()) - .with_env_var("IRONRDP_LOG") - .from_env_lossy(); - - tracing_subscriber::registry() - .with(fmt_layer) - .with(env_filter) - .try_init() - .context("failed to set tracing global subscriber")?; - - Ok(()) -} - -#[derive(Default)] -enum OptCodec { - #[default] - RemoteFX, - Bitmap, - None, - #[cfg(feature = "qoi")] - Qoi, - #[cfg(feature = "qoiz")] - QoiZ, -} - -impl core::str::FromStr for OptCodec { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "remotefx" => Ok(Self::RemoteFX), - "bitmap" => Ok(Self::Bitmap), - "none" => Ok(Self::None), - #[cfg(feature = "qoi")] - "qoi" => Ok(Self::Qoi), - #[cfg(feature = "qoiz")] - "qoiz" => Ok(Self::QoiZ), - _ => anyhow::bail!("unknown codec: {s}"), - } - } -} diff --git a/crates/ironrdp-bench/Cargo.toml b/crates/ironrdp-bench/Cargo.toml index ea35d2ceb8..d97612bf71 100644 --- a/crates/ironrdp-bench/Cargo.toml +++ b/crates/ironrdp-bench/Cargo.toml @@ -5,16 +5,50 @@ description = "IronRDP benchmarks" edition.workspace = true publish = false +[features] +default = ["qoi", "qoiz"] +qoi = ["ironrdp/qoi"] +qoiz = ["ironrdp/qoiz"] + +[dependencies] +anyhow = "1" +async-trait = "0.1" +ironrdp = { path = "../ironrdp", features = [ + "server", + "pdu", + "__bench", +] } +ironrdp-capture-replay.path = "../ironrdp-capture-replay" +pico-args = "0.5" +sha2 = "0.10" +tokio = { version = "1", features = ["sync", "fs", "time", "io-util", "macros", "rt"] } +toml = "1.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing = { version = "0.1", features = ["log"] } + [dev-dependencies] criterion = "0.8" ironrdp-graphics.path = "../ironrdp-graphics" ironrdp-pdu.path = "../ironrdp-pdu" ironrdp-server = { path = "../ironrdp-server", features = ["__bench"] } +[[bin]] +name = "perfenc" +path = "src/bin/perfenc.rs" + +[[bin]] +name = "capture-replay-bench" +path = "src/bin/capture_replay_bench.rs" + [[bench]] name = "bench" path = "benches/bench.rs" harness = false +[[bench]] +name = "capture_replay" +path = "benches/capture_replay.rs" +harness = false + [lints] workspace = true diff --git a/crates/ironrdp-bench/benches/bench.rs b/crates/ironrdp-bench/benches/bench.rs index 96c64043b9..fdf36bbdc3 100644 --- a/crates/ironrdp-bench/benches/bench.rs +++ b/crates/ironrdp-bench/benches/bench.rs @@ -1,5 +1,7 @@ #![expect(clippy::missing_panics_doc, reason = "panics in benches are allowed")] +#![allow(unused_crate_dependencies)] // The package also contains the perfenc binary. +use core::hint::black_box; use core::num::{NonZeroU16, NonZeroUsize}; use criterion::{Criterion, criterion_group, criterion_main}; @@ -8,6 +10,22 @@ use ironrdp_pdu::codecs::rfx; use ironrdp_server::BitmapUpdate; use ironrdp_server::bench::encoder::rfx::{rfx_enc, rfx_enc_tile}; +fn representative_argb(width: usize, height: usize) -> Vec { + let mut pixels = Vec::with_capacity(width * height * 4); + + for y in 0..height { + for x in 0..width { + let checker = ((x / 32) + (y / 32)) % 2; + let red = u8::try_from((x * 17 + y * 11) % 256).expect("color component fits in u8"); + let green = u8::try_from((x * 7 + y * 29) % 256).expect("color component fits in u8"); + let blue = if checker == 0 { 48 } else { 208 }; + pixels.extend_from_slice(&[blue, green, red, u8::MAX]); + } + } + + pixels +} + pub fn rfx_enc_tile_bench(c: &mut Criterion) { const WIDTH: NonZeroU16 = NonZeroU16::new(64).expect("value is guaranteed to be non-zero"); const HEIGHT: NonZeroU16 = NonZeroU16::new(64).expect("value is guaranteed to be non-zero"); @@ -22,17 +40,23 @@ pub fn rfx_enc_tile_bench(c: &mut Criterion) { width: WIDTH, height: HEIGHT, format: ironrdp_server::PixelFormat::ARgb32, - data: vec![0; 64 * 64 * 4].into(), + data: representative_argb(64, 64).into(), stride: STRIDE, }; - c.bench_function("rfx_enc_tile", |b| b.iter(|| rfx_enc_tile(&bitmap, &quant, algo, 0, 0))); + assert_ne!( + rfx_enc_tile(&bitmap, &quant, algo, 0, 0), + 0, + "tile encoder must produce output" + ); + c.bench_function("rfx_enc_tile", |b| { + b.iter(|| black_box(rfx_enc_tile(black_box(&bitmap), black_box(&quant), algo, 0, 0))) + }); } pub fn rfx_enc_bench(c: &mut Criterion) { const WIDTH: NonZeroU16 = NonZeroU16::new(2048).expect("value is guaranteed to be non-zero"); const HEIGHT: NonZeroU16 = NonZeroU16::new(2048).expect("value is guaranteed to be non-zero"); - // FIXME/QUESTION: It looks like we have a bug here, don't we? The stride value should be 2048 * 4. - const STRIDE: NonZeroUsize = NonZeroUsize::new(64 * 4).expect("value is guaranteed to be non-zero"); + const STRIDE: NonZeroUsize = NonZeroUsize::new(2048 * 4).expect("value is guaranteed to be non-zero"); let quant = rfx::Quant::default(); let algo = rfx::EntropyAlgorithm::Rlgr3; @@ -43,27 +67,46 @@ pub fn rfx_enc_bench(c: &mut Criterion) { width: WIDTH, height: HEIGHT, format: ironrdp_server::PixelFormat::ARgb32, - data: vec![0; 2048 * 2048 * 4].into(), + data: representative_argb(2048, 2048).into(), stride: STRIDE, }; - c.bench_function("rfx_enc", |b| b.iter(|| rfx_enc(&bitmap, &quant, algo))); + assert_ne!(rfx_enc(&bitmap, &quant, algo), 0, "image encoder must produce output"); + c.bench_function("rfx_enc", |b| { + b.iter(|| black_box(rfx_enc(black_box(&bitmap), black_box(&quant), algo))) + }); } pub fn to_ycbcr_bench(c: &mut Criterion) { const WIDTH: usize = 64; const HEIGHT: usize = 64; - let input = vec![0; WIDTH * HEIGHT * 4]; + let input = representative_argb(WIDTH, HEIGHT); let stride = WIDTH * 4; let mut y = [0i16; WIDTH * HEIGHT]; let mut cb = [0i16; WIDTH * HEIGHT]; let mut cr = [0i16; WIDTH * HEIGHT]; let format = ironrdp_graphics::image_processing::PixelFormat::ARgb32; + to_64x64_ycbcr_tile( + &input, + WIDTH.try_into().expect("can't panic"), + HEIGHT.try_into().expect("can't panic"), + stride.try_into().expect("can't panic"), + format, + &mut y, + &mut cb, + &mut cr, + ) + .expect("representative ARGB tile is valid"); + assert!( + y.iter().chain(&cb).chain(&cr).any(|component| *component != 0), + "color conversion must produce nonzero output" + ); + c.bench_function("to_ycbcr", |b| { b.iter(|| { to_64x64_ycbcr_tile( - &input, + black_box(&input), WIDTH.try_into().expect("can't panic"), HEIGHT.try_into().expect("can't panic"), stride.try_into().expect("can't panic"), @@ -72,7 +115,9 @@ pub fn to_ycbcr_bench(c: &mut Criterion) { &mut cb, &mut cr, ) - }) + .expect("representative ARGB tile is valid"); + black_box((y[0], cb[0], cr[0])) + }); }); } diff --git a/crates/ironrdp-bench/benches/capture_replay.rs b/crates/ironrdp-bench/benches/capture_replay.rs new file mode 100644 index 0000000000..5ef670fc94 --- /dev/null +++ b/crates/ironrdp-bench/benches/capture_replay.rs @@ -0,0 +1,27 @@ +#![allow(unused_crate_dependencies)] // The package also contains standalone benchmark binaries. + +use core::hint::black_box; + +use criterion::{Criterion, criterion_group, criterion_main}; +use ironrdp_bench::replay::{PartialReplayId, PartialReplayWorkload}; + +fn partial_replay(c: &mut Criterion) { + for id in PartialReplayId::ALL { + let name = format!("partial-replay/{}/processing", id.as_str()); + c.bench_function(&name, |b| { + let workload = PartialReplayWorkload::prepare(id).expect("qualified partial replay workload must prepare"); + workload + .verify() + .expect("qualified partial replay workload must pass strict preflight"); + b.iter(|| { + let measurement = workload + .replay() + .expect("qualified partial replay workload must execute"); + black_box(measurement) + }); + }); + } +} + +criterion_group!(benches, partial_replay); +criterion_main!(benches); diff --git a/crates/ironrdp-bench/corpus.toml b/crates/ironrdp-bench/corpus.toml index a3ef3cac60..1d31ce61b8 100644 --- a/crates/ironrdp-bench/corpus.toml +++ b/crates/ironrdp-bench/corpus.toml @@ -70,6 +70,7 @@ id = "no-nla-accepted" file = "rdp-no-nla-accepted1.pcapng" sha256 = "1146ef447ad88e58f25d14adf81581233c8265bf49140fa6f7a172fd30f3c402" intent = "RDP without NLA accepted by the server." +performance = { criterion = "partial-replay/no-nla-accepted/processing" } expect = { outcome = "partial", summary = { client_pdus = 666, server_pdus = 740, connection_pdus = 15, client_observation_pdus = 666, fast_path_pdus = 20, io_channel_pdus = 4, message_channel_pdus = 38, static_channel_pdus = 662, other_server_message_pdus = 1, graphics_updates = 78, final_dimensions = "1024x768", fingerprint = "5f280762f289c4420327a0b8b6e302ccfc3a40465a2eed9bf2dc2fa833ec7b19", lifecycle = "active", framing_gaps = 0, truncated_pdu_gaps = 0, static_channel_gaps = 8, dynamic_channel_gaps = 0, session_gaps = 0, incomplete_activation_gaps = 0, unsupported_gaps = 0, gap_fingerprint = "4b8da452635699c6de2820276840579b4f05cf92f9f636c707737312456934f1" } } [[capture]] @@ -84,6 +85,7 @@ id = "no-nla-smartcard" file = "rdp-no-nla-smartcard-auth1.pcapng" sha256 = "b29eedb9e81ed6fab7e0730951880e78fc60c3d64bfb6b37e4fa604ea3a30ff2" intent = "RDP without NLA using smartcard authentication." +performance = { criterion = "partial-replay/no-nla-smartcard/processing" } expect = { outcome = "partial", summary = { client_pdus = 903, server_pdus = 934, connection_pdus = 15, client_observation_pdus = 903, fast_path_pdus = 12, io_channel_pdus = 4, message_channel_pdus = 38, static_channel_pdus = 864, other_server_message_pdus = 1, graphics_updates = 79, final_dimensions = "1024x768", fingerprint = "51069cfd256a082e8bf9aa9d159ebc1c1d82c5f0bf7991f07d61bfa09eb81426", lifecycle = "active", framing_gaps = 0, truncated_pdu_gaps = 0, static_channel_gaps = 8, dynamic_channel_gaps = 0, session_gaps = 0, incomplete_activation_gaps = 0, unsupported_gaps = 0, gap_fingerprint = "1c6187ca6547aa904e0e752f5122a592f9eca8b68d2aa4315076536a7039a7ec" } } [[capture]] diff --git a/crates/ironrdp-bench/src/bin/capture_replay_bench.rs b/crates/ironrdp-bench/src/bin/capture_replay_bench.rs new file mode 100644 index 0000000000..9794fc713b --- /dev/null +++ b/crates/ironrdp-bench/src/bin/capture_replay_bench.rs @@ -0,0 +1,54 @@ +#![expect( + clippy::print_stderr, + reason = "the command reports qualified workload failures to its caller" +)] +#![expect( + clippy::print_stdout, + reason = "the command reports payload-free replay measurements to its caller" +)] +#![allow(unused_crate_dependencies)] // The package also contains unrelated benchmark dependencies. + +use core::str::FromStr as _; +use std::process::ExitCode; + +use ironrdp_bench::replay::{PartialReplayId, PartialReplayWorkload}; + +fn main() -> ExitCode { + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("capture replay benchmark failed: {error}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result<(), String> { + let mut arguments = pico_args::Arguments::from_env(); + if arguments.contains(["-h", "--help"]) { + if !arguments.finish().is_empty() { + return Err(usage().to_owned()); + } + println!("{}", usage()); + return Ok(()); + } + let selector: String = arguments.value_from_str("--capture").map_err(|_| usage().to_owned())?; + if !arguments.finish().is_empty() { + return Err(usage().to_owned()); + } + + let id = PartialReplayId::from_str(&selector).map_err(|error| error.to_string())?; + let workload = PartialReplayWorkload::prepare(id).map_err(|error| error.to_string())?; + let measurement = workload.verify().map_err(|error| error.to_string())?; + println!( + "workload=partial-replay/{}\trouted_pdus={}\tgraphics_updates={}", + workload.id().as_str(), + measurement.routed_pdus, + measurement.graphics_updates + ); + Ok(()) +} + +const fn usage() -> &'static str { + "usage: capture-replay-bench --capture " +} diff --git a/crates/ironrdp-bench/src/bin/perfenc.rs b/crates/ironrdp-bench/src/bin/perfenc.rs new file mode 100644 index 0000000000..3c42b81751 --- /dev/null +++ b/crates/ironrdp-bench/src/bin/perfenc.rs @@ -0,0 +1,298 @@ +#![allow(unused_crate_dependencies)] // The package also contains Criterion benchmark targets. +#![allow(clippy::print_stdout)] + +use core::num::{NonZeroU16, NonZeroUsize}; +use core::time::Duration; +use std::io; +use std::time::Instant; + +use anyhow::Context as _; +use ironrdp::pdu::rdp::capability_sets::{CmdFlags, EntropyBits, LargePointerSupportFlags}; +use ironrdp::server::bench::encoder::{UpdateEncoder, UpdateEncoderCodecs}; +use ironrdp::server::{BitmapUpdate, DesktopSize, DisplayUpdate, PixelFormat, RdpServerDisplayUpdates}; +use tokio::fs::File; +use tokio::io::AsyncReadExt as _; +use tokio::time::sleep; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + setup_logging()?; + let mut args = pico_args::Arguments::from_env(); + + if args.contains(["-h", "--help"]) { + println!("Usage: perfenc [OPTIONS] "); + println!(); + println!("Encode headerless RGBX frames with one persistent server encoder."); + println!("The default is quiet and unpaced for whole-process measurements."); + println!(); + println!("Options:"); + println!(" --width Width of the display (default: 3840)"); + println!(" --height Height of the display (default: 2400)"); + println!(" --codec Codec to use (default: remotefx)"); + println!(" Valid values: qoi, qoiz, remotefx, bitmap, none"); + println!(" --fps Limit frame delivery for interactive use (default: none)"); + return Ok(()); + } + + let width = NonZeroU16::new(args.opt_value_from_str("--width")?.unwrap_or(3840)).context("width cannot be zero")?; + let height = + NonZeroU16::new(args.opt_value_from_str("--height")?.unwrap_or(2400)).context("height cannot be zero")?; + let codec = args.opt_value_from_str("--codec")?.unwrap_or_else(OptCodec::default); + let fps = args.opt_value_from_str("--fps")?.unwrap_or(0); + + let filename: String = args.free_from_str().context("missing RGBX input filename")?; + let file = File::open(&filename) + .await + .with_context(|| format!("failed to open file: {filename}"))?; + let desktop_size = DesktopSize { + width: width.get(), + height: height.get(), + }; + let mut encoder = create_encoder(desktop_size, codec)?; + let mut updates = DisplayUpdates::new(file, width, height, fps); + + let mut total_raw = 0u64; + let mut total_encoded = 0u64; + let mut update_count = 0u64; + let mut fragment_count = 0u64; + while let Some(update) = updates.next_update().await? { + let raw_size = match &update { + DisplayUpdate::Bitmap(bitmap) => bitmap.data.len(), + _ => anyhow::bail!("RGBX source produced a non-bitmap update"), + }; + + total_raw += u64::try_from(raw_size).context("frame size does not fit in u64")?; + let mut fragments = encoder.update(update); + while let Some(fragment) = fragments.next().await { + total_encoded += + u64::try_from(fragment?.data.len()).context("encoded fragment size does not fit in u64")?; + fragment_count += 1; + } + update_count += 1; + } + + anyhow::ensure!(update_count > 0, "RGBX input contains no complete frames"); + anyhow::ensure!(fragment_count > 0, "encoder produced no output fragments"); + anyhow::ensure!(total_encoded > 0, "encoder produced no output bytes"); + + #[expect(clippy::as_conversions, reason = "u64-to-f64 conversion is used only for reporting")] + let reduction = 100.0 - (total_encoded as f64 / total_raw as f64) * 100.0; + println!( + "updates={update_count}\tfragments={fragment_count}\traw_bytes={total_raw}\tencoded_bytes={total_encoded}\treduction={reduction:.2}%" + ); + + Ok(()) +} + +fn create_encoder(desktop_size: DesktopSize, codec: OptCodec) -> anyhow::Result { + let mut flags = CmdFlags::all(); + let mut codecs = UpdateEncoderCodecs::new(); + + match codec { + OptCodec::RemoteFX => codecs.set_remotefx(Some((EntropyBits::Rlgr3, 0))), + OptCodec::Bitmap => flags -= CmdFlags::SET_SURFACE_BITS, + OptCodec::None => {} + #[cfg(feature = "qoi")] + OptCodec::Qoi => codecs.set_qoi(Some(0)), + #[cfg(feature = "qoiz")] + OptCodec::QoiZ => codecs.set_qoiz(Some(0)), + }; + + UpdateEncoder::new( + desktop_size, + flags, + codecs, + 8 * 1024 * 1024, + u16::MAX, + LargePointerSupportFlags::all(), + ) + .context("failed to initialize update encoder") +} + +struct DisplayUpdates { + file: File, + width: NonZeroU16, + height: NonZeroU16, + stride: NonZeroUsize, + fps: u32, + last_update_time: Option, +} + +impl DisplayUpdates { + fn new(file: File, width: NonZeroU16, height: NonZeroU16, fps: u32) -> Self { + Self { + file, + width, + height, + stride: NonZeroUsize::new(usize::from(width.get()) * 4) + .expect("nonzero width produces a nonzero RGBX stride"), + fps, + last_update_time: None, + } + } +} + +#[async_trait::async_trait] +impl RdpServerDisplayUpdates for DisplayUpdates { + async fn next_update(&mut self) -> ironrdp::server::ServerResult> { + use ironrdp::server::ServerErrorExt as _; + + let frame_size = self.stride.get() * usize::from(self.height.get()); + let mut frame = vec![0; frame_size]; + match read_frame(&mut self.file, &mut frame) + .await + .map_err(|error| ironrdp::server::ServerError::io("read RGBX frame", error))? + { + FrameRead::EndOfFile => return Ok(None), + FrameRead::Truncated => { + return Err(ironrdp::server::ServerError::reason("perfenc", "truncated RGBX frame")); + } + FrameRead::Complete => {} + } + + if self.fps > 0 { + let now = Instant::now(); + if let Some(last_update_time) = self.last_update_time { + let frame_interval = Duration::from_secs(1) / self.fps; + let elapsed = now - last_update_time; + if elapsed < frame_interval { + sleep(frame_interval - elapsed).await; + } + } + self.last_update_time = Some(Instant::now()); + } + + Ok(Some(DisplayUpdate::Bitmap(BitmapUpdate { + x: 0, + y: 0, + width: self.width, + height: self.height, + format: PixelFormat::RgbX32, + data: frame.into(), + stride: self.stride, + }))) + } +} + +enum FrameRead { + Complete, + EndOfFile, + Truncated, +} + +async fn read_frame(file: &mut File, frame: &mut [u8]) -> io::Result { + let mut filled = 0; + while filled < frame.len() { + let read = file.read(&mut frame[filled..]).await?; + if read == 0 { + return Ok(if filled == 0 { + FrameRead::EndOfFile + } else { + FrameRead::Truncated + }); + } + filled += read; + } + Ok(FrameRead::Complete) +} + +fn setup_logging() -> anyhow::Result<()> { + use tracing::metadata::LevelFilter; + use tracing_subscriber::EnvFilter; + use tracing_subscriber::prelude::*; + + let env_filter = EnvFilter::builder() + .with_default_directive(LevelFilter::WARN.into()) + .with_env_var("IRONRDP_LOG") + .from_env_lossy(); + + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().compact()) + .with(env_filter) + .try_init() + .context("failed to set tracing global subscriber")?; + + Ok(()) +} + +#[derive(Default)] +enum OptCodec { + #[default] + RemoteFX, + Bitmap, + None, + #[cfg(feature = "qoi")] + Qoi, + #[cfg(feature = "qoiz")] + QoiZ, +} + +impl core::str::FromStr for OptCodec { + type Err = anyhow::Error; + + fn from_str(value: &str) -> anyhow::Result { + match value { + "remotefx" => Ok(Self::RemoteFX), + "bitmap" => Ok(Self::Bitmap), + "none" => Ok(Self::None), + #[cfg(feature = "qoi")] + "qoi" => Ok(Self::Qoi), + #[cfg(feature = "qoiz")] + "qoiz" => Ok(Self::QoiZ), + _ => anyhow::bail!("unknown codec: {value}"), + } + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::{AtomicUsize, Ordering}; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + use super::*; + + static NEXT_TEMPORARY_FILE: AtomicUsize = AtomicUsize::new(0); + + fn temporary_path(name: &str) -> std::path::PathBuf { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time is after the Unix epoch") + .as_nanos(); + let sequence = NEXT_TEMPORARY_FILE.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( + "ironrdp-perfenc-{name}-{}-{timestamp}-{sequence}", + std::process::id() + )) + } + + async fn display_updates(name: &str, bytes: &[u8]) -> (DisplayUpdates, std::path::PathBuf) { + let path = temporary_path(name); + fs::write(&path, bytes).expect("write temporary frame source"); + let file = File::open(&path).await.expect("open temporary frame source"); + ( + DisplayUpdates::new( + file, + NonZeroU16::new(1).expect("one is nonzero"), + NonZeroU16::new(1).expect("one is nonzero"), + 0, + ), + path, + ) + } + + #[tokio::test] + async fn returns_none_at_clean_eof() { + let (mut updates, path) = display_updates("clean-eof", &[]).await; + assert!(updates.next_update().await.expect("clean EOF is valid").is_none()); + fs::remove_file(path).expect("remove temporary frame source"); + } + + #[tokio::test] + async fn rejects_truncated_frame() { + let (mut updates, path) = display_updates("truncated", &[0, 1, 2]).await; + let error = updates.next_update().await.expect_err("partial frame must fail"); + assert!(error.to_string().contains("truncated RGBX frame")); + fs::remove_file(path).expect("remove temporary frame source"); + } +} diff --git a/crates/ironrdp-bench/src/lib.rs b/crates/ironrdp-bench/src/lib.rs new file mode 100644 index 0000000000..1dc7338d4c --- /dev/null +++ b/crates/ironrdp-bench/src/lib.rs @@ -0,0 +1,5 @@ +//! Shared prepared workloads for IronRDP benchmarks. + +#![allow(unused_crate_dependencies)] // Package dependencies are used by standalone and Criterion targets. + +pub mod replay; diff --git a/crates/ironrdp-bench/src/replay.rs b/crates/ironrdp-bench/src/replay.rs new file mode 100644 index 0000000000..e7d0f82433 --- /dev/null +++ b/crates/ironrdp-bench/src/replay.rs @@ -0,0 +1,459 @@ +//! Prepared, qualified partial-replay workloads shared by CLI and Criterion. + +use core::str::FromStr; +use std::fmt; +use std::fs::File; +use std::path::{Path, PathBuf}; + +use ironrdp_capture_replay::{ + ReplayExecution, ReplayLifecycle, ReplayOptions, ReplaySummary, prepare_capture, read_capture, +}; +use sha2::{Digest as _, Sha256}; + +const MANIFEST: &str = include_str!("../corpus.toml"); +const CACHE_ROOT: &str = "bench-data/wireshark-rdp"; + +/// A replay workload with useful rendered output and known, qualified gaps. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PartialReplayId { + /// The no-NLA accepted capture. + NoNlaAccepted, + /// The no-NLA smartcard capture. + NoNlaSmartcard, +} + +impl PartialReplayId { + /// All partial replay workloads suitable for processing benchmarks. + pub const ALL: [Self; 2] = [Self::NoNlaAccepted, Self::NoNlaSmartcard]; + + /// Stable workload identifier shared by the CLI and Criterion. + pub const fn as_str(self) -> &'static str { + match self { + Self::NoNlaAccepted => "no-nla-accepted", + Self::NoNlaSmartcard => "no-nla-smartcard", + } + } +} + +impl FromStr for PartialReplayId { + type Err = ReplayWorkloadError; + + fn from_str(value: &str) -> ReplayWorkloadResult { + match value { + "no-nla-accepted" => Ok(Self::NoNlaAccepted), + "no-nla-smartcard" => Ok(Self::NoNlaSmartcard), + _ => Err(ReplayWorkloadError::new(format!( + "unknown partial replay workload: {value}" + ))), + } + } +} + +/// A prepared capture replay with a manifest-qualified output contract. +pub struct PartialReplayWorkload { + id: PartialReplayId, + expected: ReplaySummary, + prepared: ironrdp_capture_replay::PreparedReplay, +} + +impl PartialReplayWorkload { + /// Verify, read, decrypt, and prepare the selected cached capture. + /// + /// This intentionally runs outside focused Criterion timing. + pub fn prepare(id: PartialReplayId) -> ReplayWorkloadResult { + let cache_root = project_root().join(CACHE_ROOT); + let capture = expected_capture(id)?; + let path = cache_root.join(&capture.revision).join("captures").join(&capture.file); + verify_file(&path, &capture.sha256)?; + + let capture_data = read_capture(&path) + .map_err(|error| ReplayWorkloadError::new(format!("read capture {}: {error}", path.display())))?; + let prepared = prepare_capture(&capture_data) + .map_err(|error| ReplayWorkloadError::new(format!("prepare capture {}: {error}", path.display())))?; + + Ok(Self { + id, + expected: capture.expected, + prepared, + }) + } + + /// Execute one strict replay and enforce its complete partial-replay contract. + /// + /// This includes the full framebuffer output fingerprint and is intended for + /// preflight verification or a standalone one-run measurement. + pub fn verify(&self) -> ReplayWorkloadResult { + self.execute(true) + } + + /// Execute one fresh replay and enforce its processing contract. + /// + /// This starts with fresh session state and verifies lifecycle, routing, + /// gap, and graphics-update boundaries without hashing every framebuffer. + /// Call [`Self::verify`] before using this in a focused benchmark. + pub fn replay(&self) -> ReplayWorkloadResult { + self.execute(false) + } + + fn execute(&self, calculate_output_fingerprint: bool) -> ReplayWorkloadResult { + let execution = self + .prepared + .replay_with_options(ReplayOptions { + calculate_output_fingerprint, + }) + .map_err(|error| ReplayWorkloadError::new(format!("replay {}: {error}", self.id.as_str())))?; + validate_execution(&execution, &self.expected, calculate_output_fingerprint)?; + + Ok(ReplayMeasurement { + routed_pdus: execution.report.events.len(), + graphics_updates: execution.summary.graphics_updates, + }) + } + + /// Stable workload identifier. + pub const fn id(&self) -> PartialReplayId { + self.id + } +} + +/// Payload-free observations from one qualified replay execution. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReplayMeasurement { + /// Number of routed client and server PDUs. + pub routed_pdus: usize, + /// Number of rendered graphics updates. + pub graphics_updates: usize, +} + +/// Error returned when a cached workload cannot satisfy its declared contract. +#[derive(Debug)] +pub struct ReplayWorkloadError(String); + +impl ReplayWorkloadError { + fn new(message: String) -> Self { + Self(message) + } +} + +impl fmt::Display for ReplayWorkloadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl core::error::Error for ReplayWorkloadError {} + +/// Result returned by prepared replay workload operations. +pub type ReplayWorkloadResult = Result; + +struct ExpectedCapture { + revision: String, + file: String, + sha256: String, + expected: ReplaySummary, +} + +fn expected_capture(id: PartialReplayId) -> ReplayWorkloadResult { + expected_capture_from_manifest(MANIFEST, id) +} + +fn expected_capture_from_manifest(manifest: &str, id: PartialReplayId) -> ReplayWorkloadResult { + let manifest: toml::Table = toml::from_str(manifest) + .map_err(|error| ReplayWorkloadError::new(format!("parse corpus manifest: {error}")))?; + let upstream = required_table(&manifest, "upstream", "root")?; + let revision = required_string(upstream, "revision", "upstream")?.to_owned(); + if !is_lower_hex(&revision, 40) { + return Err(ReplayWorkloadError::new( + "corpus manifest has an invalid upstream revision".to_owned(), + )); + } + let captures = required_array(&manifest, "capture", "root")?; + let capture = captures + .iter() + .filter_map(toml::Value::as_table) + .find(|capture| capture.get("id").and_then(toml::Value::as_str) == Some(id.as_str())) + .ok_or_else(|| ReplayWorkloadError::new(format!("missing {} corpus entry", id.as_str())))?; + let expected = required_table(capture, "expect", "capture")?; + let outcome = required_string(expected, "outcome", "capture.expect")?; + if outcome != "partial" { + return Err(ReplayWorkloadError::new(format!( + "{} must remain a qualified partial replay", + id.as_str() + ))); + } + + let file = required_string(capture, "file", "capture")?.to_owned(); + if !is_safe_capture_file(&file) { + return Err(ReplayWorkloadError::new(format!("unsafe capture file name: {file}"))); + } + let sha256 = required_string(capture, "sha256", "capture")?.to_owned(); + parse_digest(&sha256)?; + + Ok(ExpectedCapture { + revision, + file, + sha256, + expected: parse_expectation(required_table(expected, "summary", "capture.expect")?)?, + }) +} + +fn parse_expectation(summary: &toml::Table) -> ReplayWorkloadResult { + Ok(ReplaySummary { + client_pdus: required_usize(summary, "client_pdus")?, + server_pdus: required_usize(summary, "server_pdus")?, + connection_pdus: required_usize(summary, "connection_pdus")?, + client_observation_pdus: required_usize(summary, "client_observation_pdus")?, + fast_path_pdus: required_usize(summary, "fast_path_pdus")?, + io_channel_pdus: required_usize(summary, "io_channel_pdus")?, + message_channel_pdus: required_usize(summary, "message_channel_pdus")?, + static_channel_pdus: required_usize(summary, "static_channel_pdus")?, + other_server_message_pdus: required_usize(summary, "other_server_message_pdus")?, + graphics_updates: required_usize(summary, "graphics_updates")?, + final_dimensions: parse_dimensions(required_string(summary, "final_dimensions", "capture.expect.summary")?)?, + output_fingerprint: Some(parse_digest(required_string( + summary, + "fingerprint", + "capture.expect.summary", + )?)?), + lifecycle: parse_lifecycle(required_string(summary, "lifecycle", "capture.expect.summary")?)?, + framing_gaps: required_usize(summary, "framing_gaps")?, + truncated_pdu_gaps: required_usize(summary, "truncated_pdu_gaps")?, + static_channel_gaps: required_usize(summary, "static_channel_gaps")?, + dynamic_channel_gaps: required_usize(summary, "dynamic_channel_gaps")?, + session_gaps: required_usize(summary, "session_gaps")?, + incomplete_activation_gaps: required_usize(summary, "incomplete_activation_gaps")?, + unsupported_gaps: required_usize(summary, "unsupported_gaps")?, + gap_fingerprint: parse_digest(required_string(summary, "gap_fingerprint", "capture.expect.summary")?)?, + }) +} + +fn validate_execution( + execution: &ReplayExecution, + expected: &ReplaySummary, + verify_output_fingerprint: bool, +) -> ReplayWorkloadResult<()> { + let mut expected = expected.clone(); + if !verify_output_fingerprint { + expected.output_fingerprint = None; + } + if execution.summary != expected + || execution.report.lifecycle != expected.lifecycle + || execution.report.events.len() + != execution + .summary + .client_pdus + .checked_add(execution.summary.server_pdus) + .ok_or_else(|| ReplayWorkloadError::new("replay PDU count overflows usize".to_owned()))? + || execution.report.gaps.len() != gap_count(&execution.summary)? + { + return Err(ReplayWorkloadError::new( + "partial replay no longer matches its declared contract".to_owned(), + )); + } + Ok(()) +} + +fn gap_count(summary: &ReplaySummary) -> ReplayWorkloadResult { + [ + summary.framing_gaps, + summary.truncated_pdu_gaps, + summary.static_channel_gaps, + summary.dynamic_channel_gaps, + summary.session_gaps, + summary.incomplete_activation_gaps, + summary.unsupported_gaps, + ] + .into_iter() + .try_fold(0usize, |count, gaps| { + count + .checked_add(gaps) + .ok_or_else(|| ReplayWorkloadError::new("replay gap count overflows usize".to_owned())) + }) +} + +fn verify_file(path: &Path, expected: &str) -> ReplayWorkloadResult<()> { + let mut file = File::open(path) + .map_err(|error| ReplayWorkloadError::new(format!("open capture {}: {error}", path.display())))?; + let mut hasher = Sha256::new(); + let mut buffer = [0; 64 * 1024]; + loop { + let read = std::io::Read::read(&mut file, &mut buffer) + .map_err(|error| ReplayWorkloadError::new(format!("hash capture {}: {error}", path.display())))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let actual: [u8; 32] = hasher.finalize().into(); + if actual != parse_digest(expected)? { + return Err(ReplayWorkloadError::new(format!( + "SHA-256 mismatch for capture {}", + path.display() + ))); + } + Ok(()) +} + +fn required_table<'a>(table: &'a toml::Table, key: &str, location: &str) -> ReplayWorkloadResult<&'a toml::Table> { + table + .get(key) + .and_then(toml::Value::as_table) + .ok_or_else(|| ReplayWorkloadError::new(format!("missing or invalid {location}.{key} table"))) +} + +fn required_array<'a>(table: &'a toml::Table, key: &str, location: &str) -> ReplayWorkloadResult<&'a Vec> { + table + .get(key) + .and_then(toml::Value::as_array) + .ok_or_else(|| ReplayWorkloadError::new(format!("missing or invalid {location}.{key} array"))) +} + +fn required_string<'a>(table: &'a toml::Table, key: &str, location: &str) -> ReplayWorkloadResult<&'a str> { + table + .get(key) + .and_then(toml::Value::as_str) + .ok_or_else(|| ReplayWorkloadError::new(format!("missing or invalid {location}.{key} string"))) +} + +fn required_usize(table: &toml::Table, key: &str) -> ReplayWorkloadResult { + let value = table + .get(key) + .and_then(toml::Value::as_integer) + .ok_or_else(|| ReplayWorkloadError::new(format!("missing or invalid capture.expect.summary.{key} integer")))?; + usize::try_from(value) + .map_err(|_| ReplayWorkloadError::new(format!("invalid capture.expect.summary.{key} integer"))) +} + +fn parse_dimensions(value: &str) -> ReplayWorkloadResult> { + if value == "-" { + return Ok(None); + } + let (width, height) = value + .split_once('x') + .ok_or_else(|| ReplayWorkloadError::new("invalid capture.expect.summary.final_dimensions".to_owned()))?; + Ok(Some(( + width + .parse() + .map_err(|_| ReplayWorkloadError::new("invalid capture.expect.summary.final_dimensions".to_owned()))?, + height + .parse() + .map_err(|_| ReplayWorkloadError::new("invalid capture.expect.summary.final_dimensions".to_owned()))?, + ))) +} + +fn parse_lifecycle(value: &str) -> ReplayWorkloadResult { + match value { + "never-activated" => Ok(ReplayLifecycle::NeverActivated), + "active" => Ok(ReplayLifecycle::Active), + "deactivated" => Ok(ReplayLifecycle::Deactivated), + _ => Err(ReplayWorkloadError::new( + "invalid capture.expect.summary.lifecycle".to_owned(), + )), + } +} + +fn parse_digest(value: &str) -> ReplayWorkloadResult<[u8; 32]> { + let mut digest = [0; 32]; + if value.len() != digest.len() * 2 { + return Err(ReplayWorkloadError::new("invalid SHA-256 digest".to_owned())); + } + for (byte, pair) in digest.iter_mut().zip(value.as_bytes().chunks_exact(2)) { + let high = match pair[0] { + b'0'..=b'9' => pair[0] - b'0', + b'a'..=b'f' => pair[0] - b'a' + 10, + _ => return Err(ReplayWorkloadError::new("invalid SHA-256 digest".to_owned())), + }; + let low = match pair[1] { + b'0'..=b'9' => pair[1] - b'0', + b'a'..=b'f' => pair[1] - b'a' + 10, + _ => return Err(ReplayWorkloadError::new("invalid SHA-256 digest".to_owned())), + }; + *byte = (high << 4) | low; + } + Ok(digest) +} + +fn is_safe_capture_file(value: &str) -> bool { + value.ends_with(".pcapng") + && value.len() > ".pcapng".len() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'.') + && !value.starts_with('.') + && !value.contains("..") +} + +fn is_lower_hex(value: &str, expected_length: usize) -> bool { + value.len() == expected_length + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn project_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("ironrdp-bench must be nested under the workspace crates directory") + .to_path_buf() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unknown_partial_replay_id() { + let error = "routing-only" + .parse::() + .expect_err("unknown workload must fail"); + assert_eq!(error.to_string(), "unknown partial replay workload: routing-only"); + } + + #[test] + fn loads_qualified_rendered_workload_contracts() { + let accepted = expected_capture(PartialReplayId::NoNlaAccepted).expect("accepted workload is declared"); + let smartcard = expected_capture(PartialReplayId::NoNlaSmartcard).expect("smartcard workload is declared"); + + assert_eq!(accepted.expected.graphics_updates, 78); + assert_eq!(smartcard.expected.graphics_updates, 79); + assert_eq!(accepted.expected.lifecycle, ReplayLifecycle::Active); + assert_eq!(smartcard.expected.lifecycle, ReplayLifecycle::Active); + assert_eq!(accepted.expected.static_channel_gaps, 8); + assert_eq!(smartcard.expected.static_channel_gaps, 8); + } + + #[test] + fn declares_every_manifest_qualified_workload() { + let manifest: toml::Table = toml::from_str(MANIFEST).expect("valid corpus manifest"); + let captures = required_array(&manifest, "capture", "root").expect("capture entries"); + let manifest_ids = captures + .iter() + .filter_map(toml::Value::as_table) + .filter(|capture| capture.contains_key("performance")) + .map(|capture| required_string(capture, "id", "capture").expect("performance capture id")) + .collect::>(); + + assert_eq!(manifest_ids, PartialReplayId::ALL.map(PartialReplayId::as_str)); + } + + #[test] + fn rejects_complete_capture_as_partial_workload() { + let manifest = MANIFEST + .replacen("id = \"clipboard-various-formats\"", "id = \"no-nla-accepted\"", 1) + .replacen("outcome = \"partial\"", "outcome = \"complete\"", 1); + assert!(expected_capture_from_manifest(&manifest, PartialReplayId::NoNlaAccepted).is_err()); + } + + #[test] + fn rejects_mismatched_replay_summary() { + let expected = expected_capture(PartialReplayId::NoNlaAccepted) + .expect("accepted workload is declared") + .expected; + let execution = ReplayExecution { + report: Default::default(), + summary: ReplaySummary::default(), + }; + assert!(validate_execution(&execution, &expected, true).is_err()); + } +} diff --git a/crates/ironrdp-bulk/benches/bulk_compression.rs b/crates/ironrdp-bulk/benches/bulk_compression.rs index ed4d0f2aa3..33355b602e 100644 --- a/crates/ironrdp-bulk/benches/bulk_compression.rs +++ b/crates/ironrdp-bulk/benches/bulk_compression.rs @@ -1,15 +1,21 @@ -//! Benchmarks for ironrdp-bulk compression algorithms. -//! -//! Measures compress + decompress throughput for MPPC (RDP4, RDP5), -//! NCRUSH (RDP6), and XCRUSH (RDP6.1) with realistic input patterns. +//! Benchmarks for supported ironrdp-bulk compression workloads. use core::hint::black_box; -use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use ironrdp_bulk::{BulkCompressor, CompressionType, flags}; -/// Text-like data (highly compressible, typical of protocol messages). -fn generate_text_data(size: usize) -> Vec { +const SUPPORTED_SIZE: usize = 4096; +const PASSTHROUGH_SIZE: usize = 16384; +const HISTORY_PACKETS: usize = 4; + +struct CompressedPacket { + bytes: Vec, + flags: u32, +} + +/// Text-like data, representative of protocol metadata and small messages. +fn text_data(size: usize) -> Vec { let phrases = [ b"Session started for user Administrator on server DESKTOP-ABC1234 ".as_slice(), b"Desktop width=1920 height=1080 bpp=32 keyboard=en-us locale=0409 ", @@ -17,129 +23,223 @@ fn generate_text_data(size: usize) -> Vec { b"Bitmap update received for region (0,0)-(1920,1080) compressed=true ", ]; let mut data = Vec::with_capacity(size); - let mut idx = 0; + let mut index = 0; + while data.len() < size { - let remaining = size - data.len(); - let phrase = phrases[idx % phrases.len()]; - let chunk = &phrase[..remaining.min(phrase.len())]; - data.extend_from_slice(chunk); - idx += 1; + let phrase = phrases[index % phrases.len()]; + data.extend_from_slice(&phrase[..(size - data.len()).min(phrase.len())]); + index += 1; } + data } -/// Structured bitmap-like data (moderately compressible - runs of similar values). -fn generate_structured_bitmap(size: usize) -> Vec { - let mut data = Vec::with_capacity(size); - // Simulate a desktop with horizontal runs of similar color - // Each "scanline" of 256 bytes has 4 color runs of 64 bytes each - let colors: [[u8; 4]; 4] = [ - [30, 60, 120, 255], // dark blue (taskbar-like) - [240, 240, 240, 255], // light gray (window background) - [0, 0, 0, 255], // black (text region) - [240, 240, 240, 255], // light gray again +/// Structured bitmap-like data with color runs and deterministic variation. +fn bitmap_data(size: usize) -> Vec { + let colors = [ + [30, 60, 120, u8::MAX], + [240, 240, 240, u8::MAX], + [0, 0, 0, u8::MAX], + [240, 240, 240, u8::MAX], ]; - let mut scanline = 0u32; - for i in 0..size { - let pos_in_scanline = i % 256; - let color_idx = pos_in_scanline / 64; - let channel = i % 4; - // Add slight variation every 4 scanlines to simulate content - let variation = if scanline.is_multiple_of(4) { - u8::try_from(pos_in_scanline & 0x03).unwrap_or(0) + let mut data = Vec::with_capacity(size); + + for index in 0..size { + let scanline_offset = index % 256; + let color = colors[scanline_offset / 64]; + let variation = if (index / 256).is_multiple_of(4) { + u8::try_from(scanline_offset & 0x03).expect("variation fits in u8") } else { 0 }; - data.push(colors[color_idx][channel].wrapping_add(variation)); - if pos_in_scanline == 255 { - scanline += 1; - } + data.push(color[index % 4].wrapping_add(variation)); } + data } -fn algo_name(ct: CompressionType) -> &'static str { - match ct { - CompressionType::Rdp4 => "mppc_rdp4", - CompressionType::Rdp5 => "mppc_rdp5", +const fn algorithm_name(compression_type: CompressionType) -> &'static str { + match compression_type { + CompressionType::Rdp4 => "mppc-rdp4", + CompressionType::Rdp5 => "mppc-rdp5", CompressionType::Rdp6 => "ncrush", CompressionType::Rdp61 => "xcrush", } } -fn bench_compress_decompress(c: &mut Criterion, ct: CompressionType, data: &[u8], label: &str) { - let name = algo_name(ct); - - // Verify data actually compresses with this algorithm - let mut test_comp = BulkCompressor::new(ct); - let (test_size, test_flags) = test_comp.compress(data).expect("bulk compression should succeed"); - let is_compressed = test_flags & flags::PACKET_COMPRESSED != 0; - - if is_compressed { - let compressed = test_comp.compressed_data(test_size).to_vec(); - - // Benchmark compress - { - let mut group = c.benchmark_group(format!("{name}/{label}")); - group.throughput(Throughput::Bytes(u64::try_from(data.len()).unwrap_or(u64::MAX))); - - group.bench_function(BenchmarkId::new("compress", data.len()), |b| { - b.iter_batched( - || BulkCompressor::new(ct), - |mut compressor| { - black_box( - compressor - .compress(black_box(data)) - .expect("bulk compression should succeed"), - ); - }, - criterion::BatchSize::SmallInput, +fn compress_packet(compressor: &mut BulkCompressor, data: &[u8]) -> CompressedPacket { + let (size, packet_flags) = compressor.compress(data).expect("bulk compression should succeed"); + assert_ne!( + packet_flags & flags::PACKET_COMPRESSED, + 0, + "supported compression workload must compress" + ); + CompressedPacket { + bytes: compressor.compressed_data(size).to_vec(), + flags: packet_flags, + } +} + +fn prepare_history(compression_type: CompressionType, data: &[u8]) -> Vec { + let mut compressor = BulkCompressor::new(compression_type); + let mut packets = Vec::with_capacity(HISTORY_PACKETS); + + for _ in 0..HISTORY_PACKETS { + packets.push(compress_packet(&mut compressor, data)); + } + + packets +} + +fn verify_round_trip(compression_type: CompressionType, packets: &[CompressedPacket], expected: &[u8]) { + let mut decompressor = BulkCompressor::new(compression_type); + + for packet in packets { + let decoded = decompressor + .decompress(&packet.bytes, packet.flags) + .expect("prepared bulk packet should decompress"); + assert_eq!(decoded, expected, "prepared bulk packet must round-trip"); + } +} + +fn bench_supported_workload(c: &mut Criterion, compression_type: CompressionType, label: &str, data: &[u8]) { + let algorithm = algorithm_name(compression_type); + let history_packets = prepare_history(compression_type, data); + verify_round_trip(compression_type, &history_packets, data); + let cold_packet = history_packets.first().expect("one cold packet is prepared"); + + let mut group = c.benchmark_group(format!("{algorithm}/{label}")); + group.throughput(Throughput::Bytes( + u64::try_from(data.len()).expect("input size fits in u64"), + )); + + group.bench_function(BenchmarkId::new("compress/cold", data.len()), |b| { + b.iter_batched_ref( + || BulkCompressor::new(compression_type), + |compressor| { + let (size, packet_flags) = compressor + .compress(black_box(data)) + .expect("bulk compression should succeed"); + assert_ne!( + packet_flags & flags::PACKET_COMPRESSED, + 0, + "supported compression workload must compress" + ); + black_box(compressor.compressed_data(size)); + black_box((size, packet_flags)) + }, + BatchSize::SmallInput, + ); + }); + group.bench_function(BenchmarkId::new("decompress/cold", data.len()), |b| { + b.iter_batched_ref( + || BulkCompressor::new(compression_type), + |decompressor| { + let decoded = decompressor + .decompress(black_box(&cold_packet.bytes), cold_packet.flags) + .expect("prepared bulk packet should decompress"); + assert_eq!( + decoded.len(), + data.len(), + "decompression must preserve the input length" ); - }); - - group.finish(); - } - - // Benchmark decompress - { - let mut group = c.benchmark_group(format!("{name}/{label}")); - group.throughput(Throughput::Bytes(u64::try_from(data.len()).unwrap_or(u64::MAX))); - - group.bench_function(BenchmarkId::new("decompress", data.len()), |b| { - b.iter_batched( - || BulkCompressor::new(ct), - |mut decompressor| { - black_box( - decompressor - .decompress(black_box(&compressed), black_box(test_flags)) - .expect("bulk decompression should succeed"), - ); - }, - criterion::BatchSize::SmallInput, + black_box(decoded); + }, + BatchSize::SmallInput, + ); + }); + group.throughput(Throughput::Bytes( + u64::try_from(data.len() * HISTORY_PACKETS).expect("history input size fits in u64"), + )); + group.bench_function(BenchmarkId::new("compress/history", data.len()), |b| { + b.iter_batched_ref( + || BulkCompressor::new(compression_type), + |compressor| { + for _ in 0..HISTORY_PACKETS { + let (size, packet_flags) = compressor + .compress(black_box(data)) + .expect("bulk compression should succeed"); + assert_ne!( + packet_flags & flags::PACKET_COMPRESSED, + 0, + "supported compression workload must compress" + ); + black_box(compressor.compressed_data(size)); + black_box((size, packet_flags)); + } + }, + BatchSize::SmallInput, + ); + }); + group.bench_function(BenchmarkId::new("decompress/history", data.len()), |b| { + b.iter_batched_ref( + || BulkCompressor::new(compression_type), + |decompressor| { + for packet in &history_packets { + let decoded = decompressor + .decompress(black_box(&packet.bytes), packet.flags) + .expect("prepared bulk packet should decompress"); + assert_eq!( + decoded.len(), + data.len(), + "decompression must preserve the input length" + ); + black_box(decoded); + } + }, + BatchSize::SmallInput, + ); + }); + + group.finish(); +} + +fn bench_passthrough(c: &mut Criterion, compression_type: CompressionType, label: &str, data: &[u8]) { + assert!(BulkCompressor::should_skip_compression(data.len())); + let algorithm = algorithm_name(compression_type); + let mut group = c.benchmark_group(format!("{algorithm}/{label}")); + group.throughput(Throughput::Bytes( + u64::try_from(data.len()).expect("input size fits in u64"), + )); + + group.bench_function(BenchmarkId::new("compress/passthrough", data.len()), |b| { + b.iter_batched_ref( + || BulkCompressor::new(compression_type), + |compressor| { + let (size, packet_flags) = compressor + .compress(black_box(data)) + .expect("bulk compression should succeed"); + assert_eq!(size, data.len(), "passthrough must preserve the input length"); + assert_eq!( + packet_flags & flags::PACKET_COMPRESSED, + 0, + "passthrough must not report compression" ); - }); + black_box((size, packet_flags)) + }, + BatchSize::SmallInput, + ); + }); - group.finish(); - } - } + group.finish(); } fn bench_all(c: &mut Criterion) { - let text_4k = generate_text_data(4096); - let text_16k = generate_text_data(16384); - let bitmap_4k = generate_structured_bitmap(4096); - let bitmap_16k = generate_structured_bitmap(16384); + let supported_text = text_data(SUPPORTED_SIZE); + let supported_bitmap = bitmap_data(SUPPORTED_SIZE); + let passthrough_text = text_data(PASSTHROUGH_SIZE); + let passthrough_bitmap = bitmap_data(PASSTHROUGH_SIZE); - for ct in [ + for compression_type in [ CompressionType::Rdp4, CompressionType::Rdp5, CompressionType::Rdp6, CompressionType::Rdp61, ] { - bench_compress_decompress(c, ct, &text_4k, "text_4k"); - bench_compress_decompress(c, ct, &text_16k, "text_16k"); - bench_compress_decompress(c, ct, &bitmap_4k, "bitmap_4k"); - bench_compress_decompress(c, ct, &bitmap_16k, "bitmap_16k"); + bench_supported_workload(c, compression_type, "text", &supported_text); + bench_supported_workload(c, compression_type, "bitmap", &supported_bitmap); + bench_passthrough(c, compression_type, "text", &passthrough_text); + bench_passthrough(c, compression_type, "bitmap", &passthrough_bitmap); } } diff --git a/crates/ironrdp-server/src/encoder/rfx.rs b/crates/ironrdp-server/src/encoder/rfx.rs index 39443bf178..12310f8996 100644 --- a/crates/ironrdp-server/src/encoder/rfx.rs +++ b/crates/ironrdp-server/src/encoder/rfx.rs @@ -235,16 +235,22 @@ pub(crate) mod bench { algo: rfx::EntropyAlgorithm, tile_x: usize, tile_y: usize, - ) { + ) -> usize { let (enc, mut data) = UpdateEncoder::new(bitmap, quant.clone(), algo); - enc.encode_tile(tile_x, tile_y, &mut data.0) + let encoded = enc + .encode_tile(tile_x, tile_y, &mut data.0) .expect("cannot propagate error in benchmark"); + encoded.y_data.len() + encoded.cb_data.len() + encoded.cr_data.len() } - pub fn rfx_enc(bitmap: &BitmapUpdate, quant: &Quant, algo: rfx::EntropyAlgorithm) { + pub fn rfx_enc(bitmap: &BitmapUpdate, quant: &Quant, algo: rfx::EntropyAlgorithm) -> usize { let (enc, mut data) = UpdateEncoder::new(bitmap, quant.clone(), algo); - enc.encode(&mut data).expect("cannot propagate error in benchmark"); + enc.encode(&mut data) + .expect("cannot propagate error in benchmark") + .iter() + .map(|tile| tile.y_data.len() + tile.cb_data.len() + tile.cr_data.len()) + .sum() } } diff --git a/xtask/README.md b/xtask/README.md index 241f2a365c..ed48204c88 100644 --- a/xtask/README.md +++ b/xtask/README.md @@ -34,5 +34,51 @@ Keep generated benchmark output under `bench-data/benchmark-output`, separate fr The ignored `bench-data` directory must never be committed. Capture-based commands must work from the verified local cache and must not perform network I/O after this explicit fetch step. +## Codec and server encoding benchmarks + +Run focused Criterion workloads with: + +```PowerShell +cargo bench -p ironrdp-bench -p ironrdp-bulk --bench bench --bench bulk_compression --locked +``` + +The graphics fixtures contain deterministic nonzero ARGB data and expose each encoder's output length. +Bulk compression measures supported 4 KiB cold and stateful-history streams separately, resetting state before every measured repeat. +Fresh codec contexts are initialized outside the timed cold and passthrough operations, while a history operation processes four packets with one context. +The separately named 16 KiB passthrough cases document the production size threshold and do not claim to measure compression. + +Build the server-encoding binary before measuring it with Hyperfine so compilation is outside the measured process: + +```PowerShell +cargo build --release -p ironrdp-bench --bin perfenc --locked +hyperfine --warmup 1 '.\target\release\perfenc.exe --width 1920 --height 1080 input.rgbx' +``` + +`perfenc` reads headerless RGBX frames, allocates and reads each frame, then encodes it with one persistent server encoder. +Its default is unpaced and emits one final payload-free summary after it confirms aggregate encoder output. +Pass `--fps ` only for interactive playback pacing. + +The capture replay workloads are intentionally qualified partial replays, not full-session success measurements. +Criterion prepares and strictly verifies only the selected cached capture before timing begins. +Each timed replay creates fresh session state and validates lifecycle, routing counters, gap metadata, and framebuffer updates without hashing every framebuffer. +The preflight verifies the full output fingerprint, while the standalone command performs that strict verification in its single replay execution. + +```PowerShell +cargo xtask bench capture-replay --capture no-nla-accepted +cargo xtask bench capture-replay --capture no-nla-smartcard +cargo build --release -p ironrdp-bench --bin capture-replay-bench --locked +hyperfine --warmup 1 '.\target\release\capture-replay-bench.exe --capture no-nla-accepted' +``` + +`cargo xtask bench capture-replay --capture ` validates the manifest eligibility and exact Criterion identity before Cargo runs. +Use direct Criterion filtering only as a lower-level local command: + +```PowerShell +cargo bench -p ironrdp-bench --bench capture_replay -- 'partial-replay/no-nla-accepted/processing' --exact +``` + +Use only `no-nla-accepted` and `no-nla-smartcard`; both are active partial replays with eight declared static-channel gaps. +Use `cargo xtask bench replay` to regression-test the complete pinned corpus, not to produce a single-capture timing score. + To update the corpus, inspect the upstream capture inventory, revise the manifest revision, inventory, scenario intent metadata, replay expectations, and SHA-256 digests together, then run `cargo xtask bench corpus-fetch` followed by `cargo xtask bench replay`. Do not commit captures, TLS key material, decrypted payloads, screenshots, or generated output. diff --git a/xtask/src/bench.rs b/xtask/src/bench.rs index 6ac5fde7ad..036a6c0b45 100644 --- a/xtask/src/bench.rs +++ b/xtask/src/bench.rs @@ -31,10 +31,19 @@ struct Capture { sha256: String, /// Recorded upstream scenario description. intent: String, + /// Optional focused performance workload for this capture. + performance: Option, /// Observed replay behavior for this exact capture revision. expect: ReplayExpectation, } +/// A focused performance workload declared by the corpus manifest. +#[derive(Clone, Debug, Eq, PartialEq)] +struct PerformanceWorkload { + /// Exact Criterion benchmark identifier. + criterion: String, +} + /// Strict expected result for one capture replay. #[derive(Clone, Debug, Eq, PartialEq)] struct ReplayExpectation { @@ -160,6 +169,42 @@ pub fn corpus_replay(selector: Option<&str>) -> anyhow::Result<()> { Ok(()) } +/// Run the manifest-declared Criterion workload for one eligible capture. +pub fn capture_replay_benchmark(selector: &str) -> anyhow::Result<()> { + let corpus = load_corpus()?; + let criterion = capture_replay_criterion(&corpus, selector)?; + let status = Command::new(env!("CARGO")) + .current_dir(project_root()) + .args([ + "bench", + "-p", + "ironrdp-bench", + "--bench", + "capture_replay", + "--locked", + "--", + ]) + .arg(criterion) + .arg("--exact") + .status() + .context("run selected capture replay benchmark")?; + anyhow::ensure!(status.success(), "selected capture replay benchmark failed: {selector}"); + Ok(()) +} + +fn capture_replay_criterion<'a>(corpus: &'a Corpus, selector: &str) -> anyhow::Result<&'a str> { + let capture = corpus + .captures + .iter() + .find(|capture| capture.id == selector) + .with_context(|| format!("unknown benchmark capture selector: {selector}"))?; + capture + .performance + .as_ref() + .map(|performance| performance.criterion.as_str()) + .with_context(|| format!("capture is not eligible for a performance benchmark: {selector}")) +} + #[derive(Debug, Eq, PartialEq)] enum ObservedReplay { Success(ReplaySummaryExpectation), @@ -347,7 +392,11 @@ fn parse_corpus(contents: &str) -> anyhow::Result { for capture in captures { let capture = capture.as_table().context("capture entry must be a TOML table")?; - ensure_allowed_keys(capture, &["id", "file", "sha256", "intent", "expect"], "capture")?; + ensure_allowed_keys( + capture, + &["id", "file", "sha256", "intent", "performance", "expect"], + "capture", + )?; let id = string(capture, "id", "capture")?.to_owned(); anyhow::ensure!(is_identifier(&id), "invalid capture identifier: {id}"); @@ -367,12 +416,32 @@ fn parse_corpus(contents: &str) -> anyhow::Result { let intent = string(capture, "intent", "capture")?.to_owned(); anyhow::ensure!(!intent.is_empty(), "capture intent must not be empty: {file}"); + let performance = capture.get("performance").map(parse_performance_workload).transpose()?; let expect = parse_replay_expectation(table(capture, "expect", "capture")?)?; + if let Some(performance) = &performance { + anyhow::ensure!( + expect.outcome == ReplayOutcome::Partial, + "performance capture must have a partial replay expectation: {id}" + ); + let summary = expect + .summary + .as_ref() + .expect("partial replay expectations always have a summary"); + anyhow::ensure!( + summary_lifecycle(summary) == Some("active"), + "performance capture must end with an active lifecycle: {id}" + ); + anyhow::ensure!( + performance.criterion == format!("partial-replay/{id}/processing"), + "invalid performance Criterion workload for capture: {id}" + ); + } parsed_captures.push(Capture { id, file, sha256, intent, + performance, expect, }); } @@ -383,6 +452,14 @@ fn parse_corpus(contents: &str) -> anyhow::Result { }) } +fn parse_performance_workload(value: &toml::Value) -> anyhow::Result { + let value = value.as_table().context("capture.performance must be a TOML table")?; + ensure_allowed_keys(value, &["criterion"], "capture.performance")?; + Ok(PerformanceWorkload { + criterion: string(value, "criterion", "capture.performance")?.to_owned(), + }) +} + const SUMMARY_KEYS: &[&str] = &[ "client_pdus", "server_pdus", @@ -711,6 +788,43 @@ expect = {{ outcome = "unsupported", stage = "negotiate", reason = "missing-rdp- assert!(error.to_string().contains("invalid capture.expect.stage string")); } + #[test] + fn selects_manifest_qualified_capture_replay_workload() { + let corpus = + parse_corpus(include_str!("../../crates/ironrdp-bench/corpus.toml")).expect("valid corpus manifest"); + + assert_eq!( + capture_replay_criterion(&corpus, "no-nla-accepted").expect("qualified capture"), + "partial-replay/no-nla-accepted/processing" + ); + assert_eq!( + capture_replay_criterion(&corpus, "no-nla-smartcard").expect("qualified capture"), + "partial-replay/no-nla-smartcard/processing" + ); + } + + #[test] + fn rejects_unknown_capture_replay_workload_before_running_cargo() { + let corpus = parse_corpus(&corpus_toml("accepted-rdp.pcapng")).expect("valid manifest"); + + let error = capture_replay_criterion(&corpus, "typo").expect_err("unknown selector must fail"); + + assert!(error.to_string().contains("unknown benchmark capture selector: typo")); + } + + #[test] + fn rejects_nonperformance_capture_replay_workload_before_running_cargo() { + let corpus = parse_corpus(&corpus_toml("accepted-rdp.pcapng")).expect("valid manifest"); + + let error = capture_replay_criterion(&corpus, "accepted-rdp").expect_err("ineligible selector must fail"); + + assert!( + error + .to_string() + .contains("capture is not eligible for a performance benchmark: accepted-rdp") + ); + } + #[test] fn rejects_unsafe_capture_file() { let error = parse_corpus(&corpus_toml("../capture.pcapng")).expect_err("unsafe path must fail"); @@ -823,6 +937,7 @@ expect = {{ outcome = "unsupported", stage = "negotiate", reason = "missing-rdp- file: "accepted-rdp.pcapng".to_owned(), sha256: SHA256_ABC.to_owned(), intent: "A direct RDP session accepted by the server.".to_owned(), + performance: None, expect: ReplayExpectation { outcome: ReplayOutcome::Unsupported, stage: Some("negotiate".to_owned()), diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs index f6fcd7d146..deb59ec848 100644 --- a/xtask/src/cli.rs +++ b/xtask/src/cli.rs @@ -12,6 +12,8 @@ TASKS: bootstrap Install all requirements for development bench corpus-fetch Fetch and verify the pinned benchmark capture corpus bench corpus-list List pinned benchmark capture corpus metadata + bench capture-replay --capture + Run one manifest-qualified capture replay Criterion workload bench replay [--capture ] Replay verified cached benchmark captures check fmt Check formatting @@ -92,6 +94,9 @@ pub enum Action { Bootstrap, BenchCorpusFetch, BenchCorpusList, + BenchCaptureReplay { + capture: String, + }, BenchReplay { capture: Option, }, @@ -161,6 +166,9 @@ pub fn parse_args() -> anyhow::Result { Some("bench") => match args.subcommand()?.as_deref() { Some("corpus-fetch") => Action::BenchCorpusFetch, Some("corpus-list") => Action::BenchCorpusList, + Some("capture-replay") => Action::BenchCaptureReplay { + capture: args.value_from_str("--capture")?, + }, Some("replay") => Action::BenchReplay { capture: args.opt_value_from_str("--capture")?, }, diff --git a/xtask/src/main.rs b/xtask/src/main.rs index d3b118a85d..a02ff16626 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -64,6 +64,7 @@ fn main() -> anyhow::Result<()> { } Action::BenchCorpusFetch => bench::corpus_fetch(&sh)?, Action::BenchCorpusList => bench::corpus_list()?, + Action::BenchCaptureReplay { capture } => bench::capture_replay_benchmark(&capture)?, Action::BenchReplay { capture } => bench::corpus_replay(capture.as_deref())?, Action::CheckFmt => check::fmt(&sh)?, Action::CheckLints => check::lints(&sh)?,