diff --git a/crates/bencher_compat/README.md b/crates/bencher_compat/README.md index cb3f65be..9ad46fca 100644 --- a/crates/bencher_compat/README.md +++ b/crates/bencher_compat/README.md @@ -72,8 +72,8 @@ $ cargo codspeed run Using codspeed-bencher-compat v1.0.0 compatibility layer NOTICE: codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment. -Checked: benches/example.rs::a (group: benches) -Checked: benches/example.rs::b (group: benches) +Checked: benches/example.rs::a (group: benches) (warmup: 5 iterations) +Checked: benches/example.rs::b (group: benches) (warmup: 5 iterations) Done running bencher_example Finished running 1 benchmark suite(s) ``` diff --git a/crates/bencher_compat/src/compat.rs b/crates/bencher_compat/src/compat.rs index 35147788..664c1962 100644 --- a/crates/bencher_compat/src/compat.rs +++ b/crates/bencher_compat/src/compat.rs @@ -1,5 +1,5 @@ use codspeed::{ - codspeed::{black_box, CodSpeed, WARMUP_RUNS}, + codspeed::{black_box, CodSpeed}, utils::{get_formated_function_path, get_git_relative_path}, }; @@ -34,13 +34,20 @@ impl Bencher { let file = get_git_relative_path(self.current_file.as_str()); let bench_path = get_formated_function_path(self.current_bench_path.as_str()); let uri = format!("{}::{}", file.to_string_lossy(), bench_path); - for _ in 0..WARMUP_RUNS { - black_box(inner()); + loop { + let warming_up = self.codspeed.warmup_iteration(); + if !warming_up { + self.codspeed.start_benchmark(uri.as_str()); + } + let output = inner(); + if !warming_up { + self.codspeed.end_benchmark(); + } + black_box(output); + if !warming_up { + break; + } } - self.codspeed.start_benchmark(uri.as_str()); - let output = inner(); - self.codspeed.end_benchmark(); - black_box(output); } } diff --git a/crates/codspeed/src/codspeed.rs b/crates/codspeed/src/codspeed.rs index 1a4644d2..9e008d88 100644 --- a/crates/codspeed/src/codspeed.rs +++ b/crates/codspeed/src/codspeed.rs @@ -1,10 +1,48 @@ use crate::measurement; use colored::Colorize; use std::ffi::CString; +use std::time::{Duration, Instant}; pub use std::hint::black_box; -pub const WARMUP_RUNS: u32 = 5; +/// Every benchmark runs at least this many unmeasured iterations before the measured one. +pub const WARMUP_MIN_ITERATIONS: u32 = 1; +/// Upper bound on unmeasured iterations, even when [`WARMUP_MAX_DURATION`] is not reached. +pub const WARMUP_MAX_ITERATIONS: u32 = 5; +/// Once the warmup has run for this long, no further unmeasured iteration is started. +/// The clock runs under the simulator, so this is simulator time, not native time. +pub const WARMUP_MAX_DURATION: Duration = Duration::from_secs(1); + +/// Progress of the unmeasured iterations that precede a measured iteration. +/// +/// Instrumentation (and therefore cache simulation) is already running while these +/// iterations execute, so the measured iteration starts with the caches populated by +/// the benchmark itself instead of with the flushed state a fresh +/// `CALLGRIND_START_INSTRUMENTATION` would leave behind. +struct Warmup { + started: Instant, + iterations: u32, +} + +impl Warmup { + fn begin() -> Self { + measurement::start_instrumentation(); + Self { + started: Instant::now(), + iterations: 0, + } + } + + fn should_iterate(&self) -> bool { + if self.iterations < WARMUP_MIN_ITERATIONS { + return true; + } + if self.iterations >= WARMUP_MAX_ITERATIONS { + return false; + } + self.started.elapsed() < WARMUP_MAX_DURATION + } +} pub fn display_native_harness() { eprintln!("Harness: codspeed v{}", env!("CARGO_PKG_VERSION"),); @@ -30,6 +68,8 @@ pub struct CodSpeed { current_benchmark: CString, group_stack: Vec, instrumentation_status: InstrumentationStatus, + warmup: Option, + warmup_iterations: u32, } impl CodSpeed { @@ -67,6 +107,8 @@ impl CodSpeed { current_benchmark: CString::new("").expect("CString::new failed"), group_stack: Vec::new(), instrumentation_status, + warmup: None, + warmup_iterations: 0, } } @@ -78,6 +120,31 @@ impl CodSpeed { self.group_stack.pop(); } + /// Returns whether the harness should run one more unmeasured iteration before the + /// measured one. The first call switches instrumentation on; the following + /// [`CodSpeed::start_benchmark`] then only zeroes the counters, keeping the simulated + /// cache state the warmup iterations built up. + /// + /// Call it before every candidate iteration: + /// + /// ```ignore + /// while codspeed.warmup_iteration() { + /// black_box(routine()); + /// } + /// codspeed.start_benchmark(uri); + /// black_box(routine()); + /// codspeed.end_benchmark(); + /// ``` + #[inline(always)] + pub fn warmup_iteration(&mut self) -> bool { + let warmup = self.warmup.get_or_insert_with(Warmup::begin); + if !warmup.should_iterate() { + return false; + } + warmup.iterations += 1; + true + } + #[inline(always)] pub fn start_benchmark(&mut self, name: &str) { self.current_benchmark = CString::new(name).expect("CString::new failed"); @@ -90,7 +157,17 @@ impl CodSpeed { // We intentionally do this no matter the instrumentation status. // The overhead in case we are not running valgrind is extremely low, and we do not want to // add a conditionnal branch to valgrind measurements. - measurement::start(); + // + // Everything executed before `zero_statistics` is discarded from the measurement, so the + // branch below is invisible to the counters. + match self.warmup.take() { + Some(warmup) => self.warmup_iterations = warmup.iterations, + None => { + self.warmup_iterations = 0; + measurement::start_instrumentation(); + } + } + measurement::zero_statistics(); } #[inline(always)] @@ -120,10 +197,11 @@ impl CodSpeed { format!(" (group: {})", self.group_stack.join("/")) }; println!( - "{}: {}{}", + "{}: {}{} (warmup: {} iterations)", action_str, self.current_benchmark.to_string_lossy(), - group_str + group_str, + self.warmup_iterations, ); } } @@ -133,3 +211,32 @@ impl Default for CodSpeed { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn warmup_runs_between_min_and_max_iterations_per_benchmark() { + let mut codspeed = CodSpeed::new(); + for _ in 0..2 { + let mut iterations = 0; + while codspeed.warmup_iteration() { + iterations += 1; + } + assert!(iterations >= WARMUP_MIN_ITERATIONS); + assert!(iterations <= WARMUP_MAX_ITERATIONS); + codspeed.start_benchmark("warmup_test"); + assert_eq!(codspeed.warmup_iterations, iterations); + codspeed.end_benchmark(); + } + } + + #[test] + fn measuring_without_warmup_reports_zero_warmup_iterations() { + let mut codspeed = CodSpeed::new(); + codspeed.start_benchmark("no_warmup"); + assert_eq!(codspeed.warmup_iterations, 0); + codspeed.end_benchmark(); + } +} diff --git a/crates/codspeed/src/measurement.rs b/crates/codspeed/src/measurement.rs index 9780e667..c614cc8a 100644 --- a/crates/codspeed/src/measurement.rs +++ b/crates/codspeed/src/measurement.rs @@ -35,10 +35,12 @@ pub fn set_metadata() { } } +/// Switches Callgrind instrumentation on. With `--cache-sim=yes` this flushes the +/// simulated cache, so it is issued once before the warmup and never between the +/// warmup and the measured iteration. #[inline(always)] -pub fn start() { +pub fn start_instrumentation() { unsafe { - send_client_request(0, &[ClientRequest::ZeroStatistics as Value, 0, 0, 0, 0, 0]); send_client_request( 0, &[ClientRequest::StartInstrumentation as Value, 0, 0, 0, 0, 0], @@ -46,6 +48,14 @@ pub fn start() { } } +/// Zeroes the cost centers without touching the simulated cache state. +#[inline(always)] +pub fn zero_statistics() { + unsafe { + send_client_request(0, &[ClientRequest::ZeroStatistics as Value, 0, 0, 0, 0, 0]); + } +} + #[inline(always)] pub fn stop(name: &CString) { unsafe { diff --git a/crates/criterion_compat/README.md b/crates/criterion_compat/README.md index 67970e8b..eafb8299 100644 --- a/crates/criterion_compat/README.md +++ b/crates/criterion_compat/README.md @@ -68,7 +68,7 @@ $ cargo codspeed run Running my_benchmark Using codspeed-criterion-compat v1.0.0 compatibility layer NOTICE: codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment. -Checked: benches/bencher_example.rs::fib_20 (group: benches) +Checked: benches/bencher_example.rs::fib_20 (group: benches) (warmup: 5 iterations) Done running bencher_example Finished running 1 benchmark suite(s) ``` diff --git a/crates/criterion_compat/src/compat/bencher.rs b/crates/criterion_compat/src/compat/bencher.rs index d4b68e12..238a42a8 100644 --- a/crates/criterion_compat/src/compat/bencher.rs +++ b/crates/criterion_compat/src/compat/bencher.rs @@ -25,13 +25,18 @@ impl<'a> Bencher<'a> { { // NOTE: this structure hardens our benchmark against dead code elimination // https://godbolt.org/z/KnYeKMd1o - for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { - if i < codspeed::codspeed::WARMUP_RUNS { - black_box(routine()); - } else { + // A single call site also keeps the warmup and the measured iteration on the same + // code addresses, so the instruction cache warmed up by the unmeasured iterations + // is the one the measured iteration uses. + loop { + let warming_up = self.codspeed.warmup_iteration(); + if !warming_up { self.codspeed.start_benchmark(self.uri.as_str()); - black_box(routine()); + } + black_box(routine()); + if !warming_up { self.codspeed.end_benchmark(); + break; } } } @@ -68,10 +73,20 @@ impl<'a> Bencher<'a> { "Note:".to_string().yellow(), self.uri.yellow(), ); - self.codspeed.start_benchmark(self.uri.as_str()); - let output = black_box(routine()); - self.codspeed.end_benchmark(); - drop(black_box(output)); + loop { + let warming_up = self.codspeed.warmup_iteration(); + if !warming_up { + self.codspeed.start_benchmark(self.uri.as_str()); + } + let output = black_box(routine()); + if !warming_up { + self.codspeed.end_benchmark(); + } + drop(black_box(output)); + if !warming_up { + break; + } + } } #[inline(never)] @@ -80,17 +95,20 @@ impl<'a> Bencher<'a> { S: FnMut() -> I, R: FnMut(I) -> O, { - for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { + loop { + let warming_up = self.codspeed.warmup_iteration(); let input = black_box(setup()); - let output = if i < codspeed::codspeed::WARMUP_RUNS { - routine(input) - } else { + if !warming_up { self.codspeed.start_benchmark(self.uri.as_str()); - let output = routine(input); + } + let output = routine(input); + if !warming_up { self.codspeed.end_benchmark(); - output - }; + } drop(black_box(output)); + if !warming_up { + break; + } } } @@ -123,18 +141,21 @@ impl<'a> Bencher<'a> { S: FnMut() -> I, R: FnMut(&mut I) -> O, { - for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { + loop { + let warming_up = self.codspeed.warmup_iteration(); let mut input = black_box(setup()); - let output = if i < codspeed::codspeed::WARMUP_RUNS { - black_box(routine(&mut input)) - } else { + if !warming_up { self.codspeed.start_benchmark(self.uri.as_str()); - let output = black_box(routine(&mut input)); + } + let output = black_box(routine(&mut input)); + if !warming_up { self.codspeed.end_benchmark(); - output - }; + } drop(black_box(output)); drop(black_box(input)); + if !warming_up { + break; + } } } @@ -162,13 +183,15 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> { { let AsyncBencher { b, runner } = self; runner.block_on(async { - for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { - if i < codspeed::codspeed::WARMUP_RUNS { - black_box(routine().await); - } else { + loop { + let warming_up = b.codspeed.warmup_iteration(); + if !warming_up { b.codspeed.start_benchmark(b.uri.as_str()); - black_box(routine().await); + } + black_box(routine().await); + if !warming_up { b.codspeed.end_benchmark(); + break; } } }); @@ -210,10 +233,20 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> { b.uri.yellow(), ); runner.block_on(async { - b.codspeed.start_benchmark(b.uri.as_str()); - let output = black_box(routine().await); - b.codspeed.end_benchmark(); - drop(black_box(output)); + loop { + let warming_up = b.codspeed.warmup_iteration(); + if !warming_up { + b.codspeed.start_benchmark(b.uri.as_str()); + } + let output = black_box(routine().await); + if !warming_up { + b.codspeed.end_benchmark(); + } + drop(black_box(output)); + if !warming_up { + break; + } + } }); } @@ -255,17 +288,20 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> { { let AsyncBencher { b, runner } = self; runner.block_on(async { - for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { + loop { + let warming_up = b.codspeed.warmup_iteration(); let input = black_box(setup()); - let output = if i < codspeed::codspeed::WARMUP_RUNS { - routine(input).await - } else { + if !warming_up { b.codspeed.start_benchmark(b.uri.as_str()); - let output = routine(input).await; + } + let output = routine(input).await; + if !warming_up { b.codspeed.end_benchmark(); - output - }; + } drop(black_box(output)); + if !warming_up { + break; + } } }) } @@ -284,18 +320,21 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> { { let AsyncBencher { b, runner } = self; runner.block_on(async { - for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { + loop { + let warming_up = b.codspeed.warmup_iteration(); let mut input = black_box(setup()); - let output = if i < codspeed::codspeed::WARMUP_RUNS { - black_box(routine(&mut input).await) - } else { + if !warming_up { b.codspeed.start_benchmark(b.uri.as_str()); - let output = black_box(routine(&mut input).await); + } + let output = black_box(routine(&mut input).await); + if !warming_up { b.codspeed.end_benchmark(); - output - }; + } drop(black_box(output)); drop(black_box(input)); + if !warming_up { + break; + } } }); } diff --git a/crates/divan_compat/README.md b/crates/divan_compat/README.md index dc448a89..848caf61 100644 --- a/crates/divan_compat/README.md +++ b/crates/divan_compat/README.md @@ -65,12 +65,12 @@ $ cargo codspeed run Collected 1 benchmark suite(s) to run Running my_benchmark NOTICE: codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment. -Checked: benches/my_benchmark.rs::fibonacci[1] -Checked: benches/my_benchmark.rs::fibonacci[2] -Checked: benches/my_benchmark.rs::fibonacci[4] -Checked: benches/my_benchmark.rs::fibonacci[8] -Checked: benches/my_benchmark.rs::fibonacci[16] -Checked: benches/my_benchmark.rs::fibonacci[32] +Checked: benches/my_benchmark.rs::fibonacci[1] (warmup: 5 iterations) +Checked: benches/my_benchmark.rs::fibonacci[2] (warmup: 5 iterations) +Checked: benches/my_benchmark.rs::fibonacci[4] (warmup: 5 iterations) +Checked: benches/my_benchmark.rs::fibonacci[8] (warmup: 5 iterations) +Checked: benches/my_benchmark.rs::fibonacci[16] (warmup: 5 iterations) +Checked: benches/my_benchmark.rs::fibonacci[32] (warmup: 5 iterations) Done running my_benchmark Finished running 1 benchmark suite(s) ``` diff --git a/crates/divan_compat/src/compat/bench/mod.rs b/crates/divan_compat/src/compat/bench/mod.rs index 5ec824b9..225b60b8 100644 --- a/crates/divan_compat/src/compat/bench/mod.rs +++ b/crates/divan_compat/src/compat/bench/mod.rs @@ -136,11 +136,21 @@ where { let mut codspeed = self.codspeed.borrow_mut(); let mut gen_input = self.config.gen_input.borrow_mut(); - let input = gen_input(); - codspeed.start_benchmark(self.uri.as_str()); - let output = benched(input); - codspeed.end_benchmark(); - divan::black_box(output); + loop { + let warming_up = codspeed.warmup_iteration(); + let input = gen_input(); + if !warming_up { + codspeed.start_benchmark(self.uri.as_str()); + } + let output = benched(input); + if !warming_up { + codspeed.end_benchmark(); + } + divan::black_box(output); + if !warming_up { + break; + } + } } pub fn bench_local_refs(self, mut benched: B) @@ -149,11 +159,20 @@ where { let mut codspeed = self.codspeed.borrow_mut(); let mut gen_input = self.config.gen_input.borrow_mut(); - let mut input = gen_input(); - - codspeed.start_benchmark(self.uri.as_str()); - let output = benched(&mut input); - codspeed.end_benchmark(); - divan::black_box(output); + loop { + let warming_up = codspeed.warmup_iteration(); + let mut input = gen_input(); + if !warming_up { + codspeed.start_benchmark(self.uri.as_str()); + } + let output = benched(&mut input); + if !warming_up { + codspeed.end_benchmark(); + } + divan::black_box(output); + if !warming_up { + break; + } + } } }