feat(divan): run each benchmark in its own process

.github/workflows/ci.yml

@@ -132,7 +132,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        partition: [1, 2, 3, 4, 5]
+        partition: [1, 2, 3, 4, 5, 6, 7, 8]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -142,7 +142,7 @@ jobs:
           bins: cargo-nextest
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/5
+      - run: cargo nextest run -p cargo-codspeed --partition hash:${{ matrix.partition }}/8
 
   analysis-integration-test:
     strategy:
@@ -160,6 +160,7 @@ jobs:
           - package: codspeed-criterion-compat
             features: async_futures
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
         with:

crates/cargo-codspeed/Cargo.toml

@@ -34,6 +34,7 @@ codspeed = { path = "../codspeed", version = "=5.0.3" }
 assert_cmd = "2.0.15"
 fs_extra = "1.3.0"
 predicates = "3.1.2"
+rstest = "0.27.0"
 
 [features]
 vendored-openssl = []

crates/cargo-codspeed/tests/process-isolation.in/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "process-isolation"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+divan = { package = "codspeed-divan-compat", path = "../../../divan_compat" }
+tokio = { version = "1.39", features = ["rt-multi-thread"] }
+
+[workspace]
+
+[[bench]]
+name = "isolated"
+harness = false
+
+[[bench]]
+name = "panicking"
+harness = false

crates/cargo-codspeed/tests/process-isolation.in/benches/isolated.rs

@@ -0,0 +1,28 @@
+use divan::Bencher;
+use std::sync::LazyLock;
+use tokio::runtime::Runtime;
+
+/// Started in `main`, before any benchmark, so its worker threads belong to
+/// the process that runs the harness.
+static RUNTIME: LazyLock<Runtime> =
+    LazyLock::new(|| Runtime::new().expect("failed to start the Tokio runtime"));
+
+#[divan::bench]
+fn first(bencher: Bencher) {
+    bencher.bench(|| divan::black_box(1));
+}
+
+#[divan::bench]
+fn second(bencher: Bencher) {
+    bencher.bench(|| divan::black_box(2));
+}
+
+#[divan::bench]
+fn spawn_on_main_runtime(bencher: Bencher) {
+    bencher.bench(|| RUNTIME.block_on(async { tokio::spawn(async { 3 }).await.unwrap() }));
+}
+
+fn main() {
+    LazyLock::force(&RUNTIME);
+    divan::main();
+}

crates/cargo-codspeed/tests/process-isolation.in/benches/panicking.rs

@@ -0,0 +1,8 @@
+#[divan::bench]
+fn panics() {
+    panic!("benchmark failed");
+}
+
+fn main() {
+    divan::main();
+}

crates/cargo-codspeed/tests/process-isolation.rs

@@ -0,0 +1,51 @@
+use predicates::str::contains;
+use rstest::rstest;
+use std::time::Duration;
+
+mod helpers;
+use helpers::*;
+
+const DIR: &str = "tests/process-isolation.in";
+/// Process isolation only applies to analysis builds.
+const MODE: [&str; 2] = ["-m", "simulation"];
+const PROCESS_ISOLATION_ENV: &str = "CODSPEED_PROCESS_ISOLATION";
+/// Exit code of a Rust process whose main thread panicked.
+const PANIC_EXIT_CODE: i32 = 101;
+/// Fails a run whose benchmark process got stuck instead of blocking the suite.
+const RUN_TIMEOUT: Duration = Duration::from_secs(120);
+
+fn build_and_run(dir: &String, bench: &str, isolation: &str) -> assert_cmd::assert::Assert {
+    cargo_codspeed(dir)
+        .args(["build", "--bench", bench])
+        .args(MODE)
+        .assert()
+        .success();
+    cargo_codspeed(dir)
+        .args(["run", "--bench", bench])
+        .args(MODE)
+        .env(PROCESS_ISOLATION_ENV, isolation)
+        .timeout(RUN_TIMEOUT)
+        .assert()
+}
+
+#[rstest]
+fn test_isolation_runs_all_benchmarks(#[values("true", "false")] isolation: &str) {
+    let dir = setup(DIR, Project::Simple);
+    build_and_run(&dir, "isolated", isolation)
+        .success()
+        .stdout(contains("isolated.rs::first"))
+        .stdout(contains("isolated.rs::second"))
+        .stdout(contains("isolated.rs::spawn_on_main_runtime"));
+    teardown(dir);
+}
+
+#[rstest]
+fn test_isolation_fails_the_run_when_a_benchmark_panics(
+    #[values("true", "false")] isolation: &str,
+) {
+    let dir = setup(DIR, Project::Simple);
+    build_and_run(&dir, "panicking", isolation)
+        .code(PANIC_EXIT_CODE)
+        .stderr(contains("benchmark failed"));
+    teardown(dir);
+}

crates/codspeed/Cargo.toml

@@ -21,7 +21,6 @@ keywords = ["codspeed", "benchmark"]
 anyhow = { workspace = true }
 colored = "3.0.0"
 glob = "0.3.2"
-libc = "^0.2"
 nix = { version = "0.31.1", features = ["time"] }
 getrandom = "0.4"
 serde = { workspace = true }

crates/codspeed/src/isolation.rs

@@ -0,0 +1,101 @@
+//! Process isolation: every benchmark runs in a fresh copy of the benchmark
+//! executable, so the allocator state and threads that other benchmarks or
+//! setup code leave behind can't change its measurement.
+
+use std::{
+    io::Write,
+    process::{Command, ExitStatus},
+    sync::LazyLock,
+};
+
+static CURRENT: LazyLock<Isolation> = LazyLock::new(Isolation::from_env);
+
+/// Isolation is on unless this is `0` or `false`.
+pub const PROCESS_ISOLATION_ENV: &str = "CODSPEED_PROCESS_ISOLATION";
+/// Set on each child process: the URI of the only benchmark it runs.
+const ISOLATED_BENCHMARK_ENV: &str = "CODSPEED_ISOLATED_BENCHMARK";
+
+/// Exit code for a child that ended with neither an exit code nor a signal.
+const GENERIC_FAILURE_EXIT_CODE: i32 = 1;
+/// Shells report a child killed by signal N as exit code 128 + N.
+#[cfg(unix)]
+const SIGNAL_EXIT_CODE_OFFSET: i32 = 128;
+
+/// How a harness runs each benchmark.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum Isolation {
+    /// All benchmarks share this process and its allocator state.
+    SharedProcess,
+    /// Every benchmark runs in its own child process.
+    ProcessPerBenchmark,
+    /// A child spawned by [`Isolation::ProcessPerBenchmark`], restricted to this URI.
+    IsolatedBenchmark(String),
+}
+
+impl Isolation {
+    /// The isolation of this process, read once from the environment.
+    pub fn current() -> &'static Self {
+        &CURRENT
+    }
+
+    fn from_env() -> Self {
+        if let Ok(uri) = std::env::var(ISOLATED_BENCHMARK_ENV) {
+            return Self::IsolatedBenchmark(uri);
+        }
+
+        match std::env::var(PROCESS_ISOLATION_ENV).as_deref() {
+            Err(_) | Ok("1" | "true") => Self::ProcessPerBenchmark,
+            Ok("0" | "false") => Self::SharedProcess,
+            Ok(other) => {
+                panic!("{PROCESS_ISOLATION_ENV} must be `true` or `false`, got `{other}`")
+            }
+        }
+    }
+
+    /// The only benchmark this process may run. It overrides CLI filters.
+    pub fn only_benchmark(&self) -> Option<&str> {
+        match self {
+            Self::IsolatedBenchmark(uri) => Some(uri),
+            Self::SharedProcess | Self::ProcessPerBenchmark => None,
+        }
+    }
+
+    /// Runs the benchmark `uri`. `bench` must contain the whole measurement,
+    /// from `start_benchmark` to `end_benchmark`.
+    pub fn run(&self, uri: String, bench: impl FnOnce(String)) {
+        match self {
+            Self::SharedProcess | Self::IsolatedBenchmark(_) => bench(uri),
+            Self::ProcessPerBenchmark => run_in_child_process(&uri),
+        }
+    }
+}
+
+/// Runs this executable again with the same arguments, limited to `uri`, and
+/// exits with the child's code if it fails.
+fn run_in_child_process(uri: &str) {
+    // Keep the parent's output before the child's.
+    if let Err(error) = std::io::stdout().flush() {
+        eprintln!("failed to flush stdout before running {uri}: {error}");
+    }
+
+    let exe = std::env::current_exe().expect("failed to locate the benchmark executable");
+    let status = Command::new(exe)
+        .args(std::env::args_os().skip(1))
+        .env(ISOLATED_BENCHMARK_ENV, uri)
+        .status()
+        .unwrap_or_else(|error| panic!("failed to start the process for {uri}: {error}"));
+
+    if !status.success() {
+        std::process::exit(exit_code(status));
+    }
+}
+
+/// Exit code of a failed child, in the shell convention.
+fn exit_code(status: ExitStatus) -> i32 {
+    #[cfg(unix)]
+    if let Some(signal) = std::os::unix::process::ExitStatusExt::signal(&status) {
+        return SIGNAL_EXIT_CODE_OFFSET + signal;
+    }
+
+    status.code().unwrap_or(GENERIC_FAILURE_EXIT_CODE)
+}

crates/codspeed/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod codspeed;
 
 pub mod instrument_hooks;
+pub mod isolation;
 
 mod macros;
 mod measurement;

crates/criterion_compat/src/compat/criterion.rs

@@ -1,6 +1,6 @@
 use std::{cell::RefCell, marker::PhantomData, rc::Rc, time::Duration};
 
-use codspeed::codspeed::CodSpeed;
+use codspeed::{codspeed::CodSpeed, isolation::Isolation};
 use criterion::{
     measurement::{Measurement, WallTime},
     profiler::Profiler,
@@ -41,6 +41,10 @@ impl Criterion {
     fn parse_filter() -> BenchmarkFilter {
         use clap::{Arg, Command};
 
+        if let Some(uri) = Isolation::current().only_benchmark() {
+            return BenchmarkFilter::Exact(uri.to_owned());
+        }
+
         let matches = Command::new("Criterion Benchmark")
             .arg(
                 Arg::new("FILTER")