From 6161dff729278b56e4687f21cf6adfd916a139e8 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 17 Nov 2026 14:20:42 +0000 Subject: [PATCH 1/6] engines-external, cli: failing tests for the ClickHouse adapter and lane - The adapter against an HTTP stub: install creates a database per install and the translated tables in it; query ids map by index and text (any language); stored steps must divide the evaluation interval and the data start; ingest sends one insert per source table in chunk order, with the series identity as sorted JSON and TSV escaping; evaluate parses series and values; exception headers, error statuses and a silent server are errors; footprint, description, CPU accounting, leftover databases, and SYNC drops. - The clickhouse lane needs its server and cgroup agent. - scripts/extbench-up.sh parses and refuses public listen addresses. - Conformance against a real server (ignored without one): one case per capability README row with hand-computed answers, the harness conformance kit, a suite-sized workload, and the lane against the fake lane. - Planner-written probe plans for the harness kit. --- Cargo.toml | 1 + aqpbm-cli/Cargo.toml | 1 + aqpbm-cli/tests/bench_external_cli.rs | 25 +- aqpbm-engines-external/Cargo.toml | 2 + .../tests/clickhouse_conformance.rs | 805 ++++++++++++++++++ .../tests/clickhouse_server.rs | 657 ++++++++++++++ .../tests/extbench_script.rs | 44 + .../tests/fixtures/plans/REV | 5 + .../fixtures/plans/probe_count_10s.plan.json | 307 +++++++ .../fixtures/plans/probe_sum_10s.plan.json | 308 +++++++ .../workloads/probe/probe_count_10s.json | 36 + .../workloads/probe/probe_sum_10s.json | 36 + 12 files changed, 2226 insertions(+), 1 deletion(-) create mode 100644 aqpbm-engines-external/tests/clickhouse_conformance.rs create mode 100644 aqpbm-engines-external/tests/clickhouse_server.rs create mode 100644 aqpbm-engines-external/tests/extbench_script.rs create mode 100644 aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json create mode 100644 aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json create mode 100644 aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json create mode 100644 aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json diff --git a/Cargo.toml b/Cargo.toml index 32b0b6a6..b666e4b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", rev = # Internal workspace crates. aqpbm-datagen = { path = "aqpbm-datagen" } aqpbm-core = { path = "aqpbm-core", default-features = false } +aqpbm-engines-external = { path = "aqpbm-engines-external" } sketch-bench = { path = "sketch-bench" } aqpbm-usecase = { path = "aqpbm-usecase" } # sketch-runtime = { path = "sketch-runtime" } # parked; diff --git a/aqpbm-cli/Cargo.toml b/aqpbm-cli/Cargo.toml index 9343ff32..e38d3eb5 100644 --- a/aqpbm-cli/Cargo.toml +++ b/aqpbm-cli/Cargo.toml @@ -28,6 +28,7 @@ heap-track = ["sketch-bench/heap-track", "aqpbm-core/heap-track"] [dependencies] aqpbm-datagen = { workspace = true, features = ["import"] } aqpbm-core.workspace = true +aqpbm-engines-external.workspace = true sketch-bench.workspace = true clap.workspace = true serde.workspace = true diff --git a/aqpbm-cli/tests/bench_external_cli.rs b/aqpbm-cli/tests/bench_external_cli.rs index d74c1c9a..6bfc18be 100644 --- a/aqpbm-cli/tests/bench_external_cli.rs +++ b/aqpbm-cli/tests/bench_external_cli.rs @@ -152,7 +152,7 @@ fn an_unknown_lane_names_the_known_lanes() { "--run-dir", run.to_str().unwrap(), "--lane", - "clickhouse", + "victoriametrics", ]); assert!(!out.status.success()); let stderr = String::from_utf8_lossy(&out.stderr); @@ -162,3 +162,26 @@ fn an_unknown_lane_names_the_known_lanes() { ); let _ = std::fs::remove_dir_all(run.parent().unwrap()); } + +/// X3: the clickhouse lane needs the server and its cgroup agent. +#[test] +fn the_clickhouse_lane_needs_its_server() { + let run = fixture("x3"); + let out = approxbench(&[ + "bench-external", + "run", + "--run-dir", + run.to_str().unwrap(), + "--lane", + "clickhouse", + "--clickhouse-url", + "http://10.0.0.2:8123", + ]); + assert!(!out.status.success()); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("--clickhouse-url") && stderr.contains("--cgroup-agent-url"), + "{stderr}" + ); + let _ = std::fs::remove_dir_all(run.parent().unwrap()); +} diff --git a/aqpbm-engines-external/Cargo.toml b/aqpbm-engines-external/Cargo.toml index 84667e0a..ef4e94a9 100644 --- a/aqpbm-engines-external/Cargo.toml +++ b/aqpbm-engines-external/Cargo.toml @@ -7,5 +7,7 @@ repository.workspace = true description = "Runs ASAPPlanner plan files on external engines: reads an InstallablePlan file (wire version 15) and translates it into ClickHouse materialized views and SQL. The translator is the capability check: a node it has no rule for is refused. Depends on no ASAPPlanner crate." [dependencies] +aqpbm-core.workspace = true serde_json.workspace = true +ureq = { version = "2", default-features = false } thiserror.workspace = true diff --git a/aqpbm-engines-external/tests/clickhouse_conformance.rs b/aqpbm-engines-external/tests/clickhouse_conformance.rs new file mode 100644 index 00000000..7da693a0 --- /dev/null +++ b/aqpbm-engines-external/tests/clickhouse_conformance.rs @@ -0,0 +1,805 @@ +//! Conformance of the ClickHouse adapter against a real server, one case +//! per row of `capabilities/README.md` (`#[ignore]`: set +//! `AQPBM_CLICKHOUSE_URL` to the server's HTTP interface and +//! `AQPBM_CGROUP_AGENT_URL` to its cgroup agent, e.g. a server started with +//! `scripts/extbench-up.sh`). Each case installs a planner-written plan +//! through the engine, ingests hand-made chunks and compares the answers +//! with hand-computed ones: (a) rows to 1e-9 relative, (b) rows by the +//! stated difference. Run with `--test-threads 1`: the cases share the +//! server. + +mod common; + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use aqpbm_engines_external::clickhouse::translate; +use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; +use aqpbm_engines_external::plan_file::PlanFile; +use aqpbm_core::harness::data::{workload_chunks, DataRecord, Sample}; +use aqpbm_core::harness::engine::{ + Answers, ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + QueryLanguage, QueryText, SampleChunk, SeriesLabels, WorkloadQuery, WorkloadSpec, +}; +use common::{fixture_path, fixture_value, fixture_value_public, lcm, payload_body}; +use serde_json::{json, Value}; + +const IGNORE: &str = + "needs a ClickHouse 26.8.39.13 server (AQPBM_CLICKHOUSE_URL, AQPBM_CGROUP_AGENT_URL)"; + +fn server(max_threads: u32) -> ClickHouseServer { + ClickHouseServer { + url: std::env::var("AQPBM_CLICKHOUSE_URL").expect("AQPBM_CLICKHOUSE_URL"), + cgroup_agent_url: std::env::var("AQPBM_CGROUP_AGENT_URL").expect("AQPBM_CGROUP_AGENT_URL"), + max_threads, + statement_timeout: Duration::from_secs(600), + } +} + +fn engine(max_threads: u32) -> ClickHouseEngine { + ClickHouseEngine::new(server(max_threads)).expect("the server answers") +} + +/// `plan` written to a temporary file, for an arm's `plan_file`. +fn plan_file(name: &str, plan: &Value) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ch-conformance-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{name}.plan.json")); + std::fs::write(&path, plan.to_string()).unwrap(); + path +} + +fn arm(path: &std::path::Path) -> ArmSpec { + ArmSpec { + name: "conformance".into(), + raw_data_retained: true, + engine_options: json!({ "plan_file": path }), + } +} + +/// A workload of `plan`'s query texts (ids `q0`, `q1`, …), evaluated every +/// `interval_ms` (the lcm of the plan's stored steps unless given), with +/// data from 0 ms. +fn workload(plan: &Value, interval_ms: Option) -> WorkloadSpec { + let texts: Vec = serde_json::from_value(plan["query_texts"].clone()).unwrap(); + let steps = translate(&PlanFile::from_value(plan.clone()).unwrap()) + .map(|d| d.query_time_steps_ms) + .unwrap_or_default(); + let interval_ms = interval_ms.unwrap_or_else(|| steps.iter().fold(10_000, |a, &b| lcm(a, b))); + WorkloadSpec { + name: "conformance".into(), + queries: texts + .iter() + .enumerate() + .map(|(i, text)| WorkloadQuery { + id: format!("q{i}"), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: text.clone(), + }], + window_ms: 60_000, + evaluation_interval_ms: interval_ms, + }) + .collect(), + data: vec![], + } +} + +fn install(engine: &ClickHouseEngine, name: &str, plan: &Value) -> Box { + engine + .install(&workload(plan, None), &arm(&plan_file(name, plan))) + .unwrap_or_else(|e| panic!("{name}: {e}")) +} + +fn series(pairs: &[(&str, &str)]) -> Arc { + Arc::new( + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) +} + +/// `samples` cut into time-ordered chunks of `chunk_ms`, ending at +/// multiples of it, time-major as the harness's chunks are. +fn chunks(mut samples: Vec, chunk_ms: i64) -> Vec { + samples.sort_by_key(|s| s.ts_ms); + let mut out: Vec = Vec::new(); + for sample in samples { + let end_ms = (sample.ts_ms + chunk_ms - 1).div_euclid(chunk_ms) * chunk_ms; + match out.last_mut() { + Some(chunk) if chunk.end_ms == end_ms => chunk.samples.push(sample), + _ => out.push(SampleChunk { + end_ms, + samples: vec![sample], + }), + } + } + out +} + +fn ingest_all(arm: &mut dyn InstalledArm, chunks: &[SampleChunk]) { + for chunk in chunks { + arm.ingest(chunk).expect("ingest"); + } +} + +/// The answers of query `q` as (labels without `__name__` joined by +/// `,`, value), for comparing with hand-written rows. +fn rows(answers: &Answers, index: usize) -> BTreeMap { + answers + .0 + .get(&format!("q{index}")) + .into_iter() + .flatten() + .map(|(labels, value)| { + let key: Vec = labels + .iter() + .filter(|(name, _)| name.as_str() != "__name__") + .map(|(name, value)| format!("{name}={value}")) + .collect(); + (key.join(","), *value) + }) + .collect() +} + +fn assert_rows(got: &BTreeMap, expected: &[(&str, f64)]) { + assert_eq!(got.len(), expected.len(), "{got:?}"); + for (key, want) in expected { + let value = got + .get(*key) + .unwrap_or_else(|| panic!("no {key} in {got:?}")); + assert!( + (value - want).abs() <= 1e-9 * want.abs().max(1.0), + "{key}: {value} vs {want}" + ); + } +} + +#[test] +#[ignore = "needs a ClickHouse server"] +fn the_server_is_the_pinned_binary() { + let _ = IGNORE; + let description: EngineDescription = engine(1).description(); + assert_eq!( + description.version, + "26.8.39.13 (d5bbb64d525197f84f9dd21489f5f6d6fdb966e5)" + ); + assert_eq!(description.build_profile, "max_threads=1"); +} + +/// Row "stored windows, exact Sum / Count / Min / Max" on F1's shape +/// `sum by (job) (_over_time(latency[1h]))` at t = 1 h: series a/1, a/2 +/// and b/1 with base 1.0, 0.5 and 2.0, value base · (1 + s mod 2) each +/// second. Per series over 3600 s: sum 5400 · base, count 3600, min base, +/// max 2 · base. So sum a 8100, b 10800; count a 7200, b 3600; min a 1.5, +/// b 2; max a 3, b 4. +#[test] +#[ignore = "needs a ClickHouse server"] +fn stored_sum_count_min_max() { + let engine = engine(1); + let samples: Vec = (1..=3600) + .flat_map(|s: i64| { + [("a", "1", 1.0), ("a", "2", 0.5), ("b", "1", 2.0)] + .into_iter() + .map(move |(job, i, base)| Sample { + labels: series(&[("__name__", "latency"), ("job", job), ("i", i)]), + ts_ms: s * 1000, + value: base * (1 + s % 2) as f64, + }) + }) + .collect(); + let chunks = chunks(samples, 60_000); + for (kind, expected) in [ + ("Sum", [("job=a", 8100.0), ("job=b", 10800.0)]), + ("Count", [("job=a", 7200.0), ("job=b", 3600.0)]), + ("Min", [("job=a", 1.5), ("job=b", 2.0)]), + ("Max", [("job=a", 3.0), ("job=b", 4.0)]), + ] { + let mut plan = fixture_value("sot_1h_stored_sum"); + payload_body(&mut plan, 1)["summary_type"] = json!({"ExactAggregate": [kind, kind]}); + let mut arm = install(&engine, &format!("stored_{kind}"), &plan); + ingest_all(arm.as_mut(), &chunks); + let (answers, _) = arm.evaluate(3_600_000).unwrap(); + assert_rows(&rows(&answers, 0), &expected); + let footprint = arm.footprint().unwrap(); + println!("stored {kind}: {:?} {footprint:?}", rows(&answers, 0)); + } +} + +/// The counter data of the translator tests (x, y, z, w), each series in its +/// own job, over 600 s. +fn counter_samples() -> Vec { + let mut samples = Vec::new(); + for s in 1..=600i64 { + let x = if s <= 450 { s } else { s - 450 } as f64; + let y = if s <= 360 { 3 * s } else { s - 360 } as f64; + let z = if s < 400 { + s as f64 + } else if s < 500 { + (s - 5) as f64 + } else { + (s - 500) as f64 / 2.0 + }; + for (job, value) in [("x", x), ("y", y), ("z", z)] { + samples.push(Sample { + labels: series(&[("__name__", "requests_total"), ("job", job)]), + ts_ms: s * 1000, + value, + }); + } + if s % 10 == 5 { + samples.push(Sample { + labels: series(&[("__name__", "requests_total"), ("job", "w")]), + ts_ms: s * 1000, + value: s as f64, + }); + } + } + samples +} + +/// Row "stored windows, exact Rate / Increase" (b). Three installs take the +/// 10 s chunks in turn, as `run_arm` feeds its installs, at max_threads 1 +/// and 16. Each answers PromQL's rate minus the first value after each +/// reset (rates are increase / 299 here; see the translator's +/// `stored_rate_differs_from_promql_by_the_first_value_after_each_reset`): +/// x 298/299, y 416/299, z 247/299, w 1. +#[test] +#[ignore = "needs a ClickHouse server"] +fn stored_rate_is_promql_minus_the_first_value_after_each_reset() { + let chunks = chunks(counter_samples(), 10_000); + let expected = [ + ("job=x", 298.0 / 299.0), + ("job=y", 416.0 / 299.0), + ("job=z", 247.0 / 299.0), + ("job=w", 1.0), + ]; + for threads in [1, 16] { + let engine = engine(threads); + let plan = fixture_value("rate_5m_stored"); + let mut installs: Vec<_> = (0..3) + .map(|i| install(&engine, &format!("rate_stored_{threads}_{i}"), &plan)) + .collect(); + for chunk in &chunks { + for arm in installs.iter_mut() { + arm.ingest(chunk).unwrap(); + } + } + for arm in installs.iter_mut() { + let (answers, _) = arm.evaluate(600_000).unwrap(); + assert_rows(&rows(&answers, 0), &expected); + } + assert_eq!( + installs[0].plan_summary().substitutions.len(), + 1, + "the substitute is recorded" + ); + } +} + +/// Row "query-time Rate / Increase" (a): the same data, PromQL's rates +/// x 1, y 417/299, z 642/299, w 1. +#[test] +#[ignore = "needs a ClickHouse server"] +fn query_time_rate_is_promql() { + let engine = engine(1); + let mut arm = install( + &engine, + "rate_query_time", + &fixture_value("rate_5m_query_time"), + ); + ingest_all(arm.as_mut(), &chunks(counter_samples(), 10_000)); + let (answers, _) = arm.evaluate(600_000).unwrap(); + assert_rows( + &rows(&answers, 0), + &[ + ("job=x", 1.0), + ("job=y", 417.0 / 299.0), + ("job=z", 642.0 / 299.0), + ("job=w", 1.0), + ], + ); + // Row "raw samples retained": the source table keeps rows, no view. + let footprint = arm.footprint().unwrap(); + assert!( + footprint.raw_retained_bytes > 0 && footprint.view_bytes == 0, + "{footprint:?}" + ); + println!("query-time rate footprint {footprint:?}"); +} + +/// Rows "query-time Sum … grouped" and "Aggregate": q4c at 1 m over job a +/// (1.0 and 0.5 each second) and b (2.0): at 120 s, a 90, b 120. q1 (0.99 +/// quantile per series over 1 m) of values 1..60: 1 + 0.99 · 59 = 59.41. +/// q3 (avg by job of the 0.9 quantiles) of two series 1..60 and 2·(1..60): +/// (54.1 + 108.2) / 2 = 81.15. F4's max of j3 = s over (300 s, 600 s]: 600. +#[test] +#[ignore = "needs a ClickHouse server"] +fn grouped_sums_quantiles_and_aggregates() { + let engine = engine(1); + let latency = + |rows: &[(&str, &str, f64)], seconds: std::ops::RangeInclusive, step: bool| { + seconds + .flat_map(|s| { + rows.iter().map(move |(job, id, base)| Sample { + labels: series(&[("__name__", "latency"), ("job", job), ("id", id)]), + ts_ms: s * 1000, + value: if step { base * (s - 60) as f64 } else { *base }, + }) + }) + .collect::>() + }; + let mut q4c = install( + &engine, + "q4c", + &fixture_value_public( + "q4c_sum_over_time__1m_1hz_10s", + "clickhouse.equivalent-only", + ), + ); + ingest_all( + q4c.as_mut(), + &chunks( + latency( + &[("a", "1", 1.0), ("a", "2", 0.5), ("b", "1", 2.0)], + 1..=120, + false, + ), + 10_000, + ), + ); + assert_rows( + &rows(&q4c.evaluate(120_000).unwrap().0, 0), + &[("job=a", 90.0), ("job=b", 120.0)], + ); + + // Values 1..60 over (60 s, 120 s]. + let ramp = chunks( + latency(&[("a", "1", 1.0), ("a", "2", 2.0)], 61..=120, true), + 10_000, + ); + let mut q1 = install( + &engine, + "q1", + &fixture_value_public("q1_quantile__1m_1hz_10s", "clickhouse.equivalent-only"), + ); + ingest_all(q1.as_mut(), &ramp); + let answers = q1.evaluate(120_000).unwrap().0; + assert_rows( + &rows(&answers, 0), + &[("id=1,job=a", 59.41), ("id=2,job=a", 2.0 * 59.41)], + ); + let mut q3 = install( + &engine, + "q3", + &fixture_value_public("q3_avg_quantile__1m_1hz_10s", "clickhouse.equivalent-only"), + ); + ingest_all(q3.as_mut(), &ramp); + assert_rows( + &rows(&q3.evaluate(120_000).unwrap().0, 0), + &[("job=a", 81.15)], + ); + + let mut f4 = install(&engine, "f4", &fixture_value("max_5m_label_filter")); + let samples: Vec = (1..=600) + .flat_map(|s: i64| { + [("j3", 1.0), ("j4", 10.0)] + .into_iter() + .map(move |(job, m)| Sample { + labels: series(&[("__name__", "latency"), ("job", job)]), + ts_ms: s * 1000, + value: m * s as f64, + }) + }) + .collect(); + ingest_all(f4.as_mut(), &chunks(samples, 300_000)); + assert_rows( + &rows(&f4.evaluate(600_000).unwrap().0, 0), + &[("job=j3", 600.0)], + ); +} + +/// Row "instant selector": rangeheavy's `sum by (job) (data)` with the +/// plan's 5 ms lookback at t = 120 s: s1 (a) at t − 10 ms (100) and t − 3 ms +/// (1); s2 (a) only at t − 5 ms, outside (t − 5 ms, t]; s3 (a) at t (2); s4 +/// (b) at t − 1 ms (4): a 3, b 4. +#[test] +#[ignore = "needs a ClickHouse server"] +fn instant_selector() { + let engine = engine(1); + let plan = fixture_value_public("rangeheavy", "clickhouse.equivalent-only"); + let mut arm = install(&engine, "rangeheavy", &plan); + let t = 120_000; + let samples = [ + (10, 100.0, "a", "s1"), + (5, 50.0, "a", "s2"), + (3, 1.0, "a", "s1"), + (1, 4.0, "b", "s4"), + (0, 2.0, "a", "s3"), + ] + .into_iter() + .map(|(before, value, job, id)| Sample { + labels: series(&[("__name__", "data"), ("job", job), ("id", id)]), + ts_ms: t - before, + value, + }) + .collect(); + arm.ingest(&SampleChunk { end_ms: t, samples }).unwrap(); + let answers = arm.evaluate(t).unwrap().0; + assert_rows(&rows(&answers, 0), &[("job=a", 3.0), ("job=b", 4.0)]); +} + +/// Row "Aggregate top_k": rates 1..5 for i1..i5, NaN for i6 and 5 for i7. +/// k = 2 keeps i5 and i7 (equal values: the label order breaks the tie); +/// k = 6 keeps every series but the NaN one. +#[test] +#[ignore = "needs a ClickHouse server"] +fn top_k_ties_and_nan() { + let engine = engine(1); + let samples: Vec = (1..=600) + .flat_map(|s: i64| { + [1.0, 2.0, 3.0, 4.0, 5.0, f64::NAN, 5.0] + .into_iter() + .enumerate() + .map(move |(i, rate)| Sample { + labels: series(&[ + ("__name__", "requests_total"), + ("instance", &format!("i{}", i + 1)), + ]), + ts_ms: s * 1000, + value: rate * s as f64, + }) + }) + .collect(); + let chunks = chunks(samples, 60_000); + for (k, expected) in [ + (2, vec![("instance=i5", 5.0), ("instance=i7", 5.0)]), + ( + 6, + vec![ + ("instance=i1", 1.0), + ("instance=i2", 2.0), + ("instance=i3", 3.0), + ("instance=i4", 4.0), + ("instance=i5", 5.0), + ("instance=i7", 5.0), + ], + ), + ] { + let mut plan = fixture_value_public("topk__5m__1hz__60s", "clickhouse.equivalent-only"); + payload_body(&mut plan, 5)["measures"][0]["k"] = json!(k); + let mut arm = install(&engine, &format!("topk_{k}"), &plan); + ingest_all(arm.as_mut(), &chunks); + assert_rows(&rows(&arm.evaluate(600_000).unwrap().0, 0), &expected); + } +} + +/// Rows "query-time … regrouping merges" and "Project / BinaryOp": +/// mixed-03 root 11, `avg by (region, service) (quantile_over_time(0.9, +/// data[100s]))` as sum / count, at 120 s over constant series (j1, r1) 1, +/// (j2, r1) 3, (j1, r2) 5: r1/s1 2, r2/s1 5. Two p90s of 1.5e308 overflow +/// the sum: the checked division is an error. +#[test] +#[ignore = "needs a ClickHouse server"] +fn average_as_sum_over_count_and_checked_division() { + let engine = engine(1); + let plan = fixture_value_public("mixed-03", "clickhouse.equivalent-only"); + let constant = |rows: &[(&str, &str, f64)]| { + (1..=120) + .flat_map(|s: i64| { + rows.iter().map(move |(job, region, value)| Sample { + labels: series(&[ + ("__name__", "data"), + ("job", job), + ("region", region), + ("service", "s1"), + ]), + ts_ms: s * 1000, + value: *value, + }) + }) + .collect::>() + }; + let mut arm = install(&engine, "mixed03", &plan); + ingest_all( + arm.as_mut(), + &chunks( + constant(&[("j1", "r1", 1.0), ("j2", "r1", 3.0), ("j1", "r2", 5.0)]), + 30_000, + ), + ); + let answers = arm.evaluate(120_000).unwrap().0; + assert_rows( + &rows(&answers, 3), + &[("region=r1,service=s1", 2.0), ("region=r2,service=s1", 5.0)], + ); + let mut overflow = install(&engine, "mixed03_overflow", &plan); + ingest_all( + overflow.as_mut(), + &chunks( + constant(&[("j1", "r1", 1.5e308), ("j2", "r1", 1.5e308)]), + 30_000, + ), + ); + let error = overflow.evaluate(120_000).err().expect("an error"); + assert!(error.0.contains("checked division"), "{error}"); +} + +/// Rows of class (c): IRate and a query-time result kept across +/// evaluations are refused at install. +#[test] +#[ignore = "needs a ClickHouse server"] +fn class_c_rows_are_refused() { + let engine = engine(1); + let mut irate = fixture_value("rate_5m_query_time"); + payload_body(&mut irate, 2)["summary_type"] = json!({"ExactAggregate": ["IRate", "IRate"]}); + let error = engine + .install(&workload(&irate, None), &arm(&plan_file("irate", &irate))) + .err() + .expect("refused"); + assert!(error.0.contains("exact IRate"), "{error}"); + let mut kept = fixture_value("joint_5m"); + kept["dag"]["nodes"][1]["lifetime"] = json!({"AcrossEvaluations": "Memory"}); + let error = engine + .install(&workload(&kept, None), &arm(&plan_file("kept", &kept))) + .err() + .expect("refused"); + assert!(error.0.contains("kept across evaluations"), "{error}"); +} + +/// Rows "raw samples retained" (a stored-only metric keeps no rows) and +/// "retention" (the TTL statements apply on the server). +#[test] +#[ignore = "needs a ClickHouse server"] +fn storage_and_retention() { + let engine = engine(1); + let plan = fixture_value("sot_1h_stored_sum"); + let mut arm = install(&engine, "sot_storage", &plan); + let samples: Vec = (1..=120) + .map(|s: i64| Sample { + labels: series(&[("__name__", "latency"), ("job", "a")]), + ts_ms: s * 1000, + value: 1.0, + }) + .collect(); + ingest_all(arm.as_mut(), &chunks(samples, 60_000)); + let footprint = arm.footprint().unwrap(); + assert!( + footprint.raw_retained_bytes == 0 && footprint.view_bytes > 0, + "{footprint:?}" + ); + println!("stored-only footprint {footprint:?}"); + + let deployment = translate(&PlanFile::from_value(plan).unwrap()).unwrap(); + let url = server(1).url; + let http = |sql: &str| { + ureq::post(&format!("{url}/")) + .query("database", "conformance_ttl") + .send_string(sql) + .map(|r| r.into_string().unwrap()) + }; + ureq::post(&format!("{url}/")) + .send_string("CREATE DATABASE IF NOT EXISTS conformance_ttl") + .unwrap(); + for statement in deployment.tables.iter().chain(&deployment.ttl_statements()) { + http(statement).unwrap_or_else(|e| panic!("{statement}: {e}")); + } + let created = http("SELECT create_table_query FROM system.tables WHERE database = 'conformance_ttl' AND name = 'v1'").unwrap(); + assert!(created.contains("TTL"), "{created}"); + ureq::post(&format!("{url}/")) + .send_string("DROP DATABASE conformance_ttl SYNC") + .unwrap(); +} + +/// The harness kit on an engine that names the planner-written probe plans +/// (`sum_over_time(probe[10s])`, `count_over_time(probe[10s])`): answers +/// are deterministic and windows are left-open. +#[test] +#[ignore = "needs a ClickHouse server"] +fn the_harness_conformance_kit_passes() { + struct WithProbePlans(ClickHouseEngine); + impl EngineUnderTest for WithProbePlans { + fn description(&self) -> EngineDescription { + self.0.description() + } + fn cpu_accounting(&self) -> CpuAccounting { + self.0.cpu_accounting() + } + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let plan = match workload.name.as_str() { + "probe-sum" => "probe_sum_10s", + "probe-count" => "probe_count_10s", + other => return Err(EngineError(format!("no probe plan for {other}"))), + }; + let mut arm = arm.clone(); + arm.engine_options = json!({ "plan_file": fixture_path(plan) }); + self.0.install(workload, &arm) + } + } + let mut engine = WithProbePlans(engine(1)); + let clock = aqpbm_core::harness::protocol::OsCpuClock; + let clocks = aqpbm_core::harness::protocol::MeasurementClocks::with_calling_thread(&clock); + let report = aqpbm_core::harness::conformance::check(&mut engine, &clocks); + println!("harness kit: {report:?}"); + let passed = report.expect("the kit passes").passed; + assert!(passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::DeterministicAnswers)); + assert!(passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::LeftOpenWindow)); +} + +/// Suite size: 1,000 series in 10 jobs, 1 s samples for 10 min from +/// 1,700,000,000,000 ms, q4c at 1 m evaluated every 10 s. Every chunk is one +/// insert of 10,000 rows; at the last evaluation each job's answer equals +/// the sum, computed here, of its samples in the last minute. The server +/// cgroup's CPU grows over the ingest and is never throttled. +#[test] +#[ignore = "needs a ClickHouse server"] +fn suite_sized_workload() { + let engine = engine(1); + let start_ms: i64 = 1_700_000_000_000; + let record: DataRecord = serde_json::from_value(json!({ + "metric": "latency", + "description": { + "labels": [{"name": "job", "cardinality": 10}, {"name": "instance", "cardinality": 100}], + "series_num": 1000, "start_ms": start_ms, "sample_interval_ms": 1000, + "duration_ms": 600000, + "value_distribution": {"kind": "uniform", "lower_bound": 0.0, "upper_bound": 10.0, "seed": 0}, + "counter": false, "seed": 7 + } + })) + .unwrap(); + let plan = fixture_value_public( + "q4c_sum_over_time__1m_1hz_10s", + "clickhouse.equivalent-only", + ); + let mut spec = workload(&plan, Some(10_000)); + spec.data = vec![record.clone()]; + let mut arm = engine + .install(&spec, &arm(&plan_file("suite", &plan))) + .expect("installs"); + let CpuAccounting::Cgroup(source) = engine.cpu_accounting() else { + panic!("cgroup accounting"); + }; + let cpu_before = source.cpu().expect("the agent answers"); + let mut last_minute: BTreeMap = BTreeMap::new(); + let end_ms = start_ms + 600_000; + let ingest_started = std::time::Instant::now(); + let mut rows_per_chunk = Vec::new(); + for chunk in workload_chunks(&[record], 10_000).unwrap() { + let chunk = chunk.unwrap(); + rows_per_chunk.push(chunk.samples.len()); + for sample in &chunk.samples { + if sample.ts_ms > end_ms - 60_000 { + *last_minute.entry(sample.labels["job"].clone()).or_default() += sample.value; + } + } + arm.ingest(&chunk).unwrap(); + if chunk.end_ms >= start_ms + 60_000 { + arm.evaluate(chunk.end_ms).unwrap(); + } + } + let ingest_wall = ingest_started.elapsed(); + let cpu_after = source.cpu().unwrap(); + let answers = arm.evaluate(end_ms).unwrap().0; + let got = rows(&answers, 0); + let expected: Vec<(String, f64)> = last_minute + .iter() + .map(|(job, sum)| (format!("job={job}"), *sum)) + .collect(); + let expected: Vec<(&str, f64)> = expected.iter().map(|(k, v)| (k.as_str(), *v)).collect(); + assert_rows(&got, &expected); + assert!( + rows_per_chunk.iter().all(|&n| n == 10_000), + "{rows_per_chunk:?}" + ); + assert!(cpu_after.usage_ns > cpu_before.usage_ns); + assert_eq!(cpu_after.nr_throttled, cpu_before.nr_throttled); + println!( + "suite size: 60 chunks of 10,000 rows, 55 evaluations; wall {:.1} s; server cgroup CPU {:.3} s", + ingest_wall.as_secs_f64(), + (cpu_after.usage_ns - cpu_before.usage_ns) as f64 / 1e9 + ); + // The idle server's CPU over 10 s. + let idle_before = source.cpu().unwrap(); + std::thread::sleep(Duration::from_secs(10)); + let idle_after = source.cpu().unwrap(); + println!( + "idle server cgroup CPU over 10 s: {:.1} ms", + (idle_after.usage_ns - idle_before.usage_ns) as f64 / 1e6 + ); +} + +/// The lane end to end: the probe workloads on lane `fake` and lane +/// `clickhouse` over one run directory, reported with the fake as truth: +/// every point is there, and none differs. +#[test] +#[ignore = "needs a ClickHouse server"] +fn the_clickhouse_lane_matches_the_fake_lane() { + use aqpbm_core::harness::bench_external::{report, run_lane, LaneSetup}; + use aqpbm_core::harness::fake::FakeEngine; + use aqpbm_core::harness::protocol::{ + load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, + }; + use aqpbm_core::harness::rundir::RunDir; + + let root = std::env::temp_dir().join(format!("ch-lane-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let run_dir = RunDir::create_at(&root).unwrap(); + let write = |path: PathBuf, text: String| { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); + }; + for workload in aqpbm_core::harness::conformance::probe_workloads() { + let plan = match workload.name.as_str() { + "probe-sum" => "probe_sum_10s", + _ => "probe_count_10s", + }; + write( + run_dir + .path() + .join(format!("workloads/{}.json", workload.name)), + serde_json::to_string(&workload).unwrap(), + ); + write( + run_dir + .path() + .join(format!("plans/equivalent-only/{}.plan.json", workload.name)), + std::fs::read_to_string(fixture_path(plan)).unwrap(), + ); + } + write( + run_dir.path().join("plans/equivalent-only/variant.json"), + r#"{"substitutions_allowed": false}"#.into(), + ); + let protocol = Protocol { + warmup_runs: 1, + measured_runs: 3, + max_core_busy_fraction: 1.0, + }; + let clock = OsCpuClock; + let clocks = MeasurementClocks::with_calling_thread(&clock); + let fake = FakeEngine::default(); + let clickhouse = engine(1); + for lane in [&fake as &dyn EngineUnderTest, &clickhouse] { + let mut probe = load_probe_for(&lane.cpu_accounting(), &ProcessCpuFiles::os()).unwrap(); + let cells = run_lane( + &run_dir, + lane, + None, + LaneSetup { + protocol: &protocol, + core: 0, + probe: probe.as_mut(), + clocks: &clocks, + args: vec![], + }, + ) + .unwrap(); + println!("{} cells: {cells:?}", lane.description().id); + } + let report = report(&run_dir, "fake", "equivalent-only").unwrap(); + println!("{}", report.summary_markdown()); + let csv = report.summary_csv(); + let clickhouse_rows: Vec<&str> = csv.lines().filter(|l| l.contains(",clickhouse,")).collect(); + assert_eq!(clickhouse_rows.len(), 2, "{csv}"); + let accuracy: Vec<_> = report + .accuracy + .iter() + .filter(|a| a.lane == "clickhouse") + .collect(); + assert_eq!(accuracy.len(), 2, "{:?}", report.accuracy); + for query in accuracy { + let errors = &query.errors; + assert!(errors.points > 0, "{query:?}"); + assert_eq!(errors.missing_points, 0, "{query:?}"); + assert_eq!(errors.extra_points, 0, "{query:?}"); + assert!(errors.max_rel_error.unwrap_or(0.0) <= 1e-9, "{query:?}"); + } + let _ = std::fs::remove_dir_all(&root); +} diff --git a/aqpbm-engines-external/tests/clickhouse_server.rs b/aqpbm-engines-external/tests/clickhouse_server.rs new file mode 100644 index 00000000..c7b56d0b --- /dev/null +++ b/aqpbm-engines-external/tests/clickhouse_server.rs @@ -0,0 +1,657 @@ +//! S1–S10: the ClickHouse engine adapter against an in-test HTTP stub that +//! records every request and answers by rules. No server is needed. + +mod common; + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use aqpbm_engines_external::clickhouse_server::{ + series_identity, ClickHouseEngine, ClickHouseServer, +}; +use aqpbm_core::harness::cgroup::CgroupSource; +use aqpbm_core::harness::data::{DataRecord, Sample}; +use aqpbm_core::harness::engine::{ + ArmSpec, CpuAccounting, EngineUnderTest, QueryLanguage, QueryText, SampleChunk, SeriesLabels, + WorkloadQuery, WorkloadSpec, +}; +use common::{fixture_path, fixture_value, payload_body}; +use serde_json::json; + +/// One request the stub received. +#[derive(Debug, Clone)] +struct Request { + params: BTreeMap, + body: String, +} + +impl Request { + /// The statement: the `query` parameter of an insert, else the body. + fn statement(&self) -> &str { + self.params + .get("query") + .map(String::as_str) + .unwrap_or(&self.body) + } +} + +/// An answer: status, extra headers, body. +type Reply = (u16, Vec<(String, String)>, String); +type Rule = dyn Fn(&Request) -> Option + Send + Sync; + +/// The stub: answers the version and leftover-database questions, then the +/// first rule that matches, else `200 ""`. +struct Stub { + url: String, + requests: Arc>>, +} + +fn start_stub(rules: Vec>) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&requests); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let Some(request) = read_request(&mut stream) else { + continue; + }; + seen.lock().unwrap().push(request.clone()); + let (status, headers, body) = rules + .iter() + .find_map(|rule| rule(&request)) + .or_else(|| default_reply(&request)) + .unwrap_or((200, vec![], String::new())); + let mut head = format!( + "HTTP/1.1 {status} X\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + for (name, value) in headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + let _ = stream.write_all(format!("{head}\r\n{body}").as_bytes()); + } + }); + Stub { url, requests } +} + +fn default_reply(request: &Request) -> Option { + let statement = request.statement(); + if statement.contains("SELECT version()") { + return Some((200, vec![], "26.8.39.13\td5bbb64d\n".into())); + } + if statement.contains("system.databases") { + return Some((200, vec![], String::new())); + } + None +} + +fn read_request(stream: &mut std::net::TcpStream) -> Option { + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).ok()?; + let target = line.split_whitespace().nth(1)?.to_string(); + let mut length = 0; + loop { + let mut header = String::new(); + reader.read_line(&mut header).ok()?; + if header == "\r\n" || header.is_empty() { + break; + } + if let Some((name, value)) = header.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + length = value.trim().parse().ok()?; + } + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).ok()?; + let params = target + .split_once('?') + .map(|(_, query)| query) + .unwrap_or("") + .split('&') + .filter(|pair| !pair.is_empty()) + .map(|pair| { + let (name, value) = pair.split_once('=').unwrap_or((pair, "")); + (decode(name), decode(value)) + }) + .collect(); + Some(Request { + params, + body: String::from_utf8(body).ok()?, + }) +} + +/// URL form decoding: `+` is a space, `%XX` a byte. +fn decode(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'+' => out.push(b' '), + b'%' if i + 2 < bytes.len() => { + out.push(u8::from_str_radix(&text[i + 1..i + 3], 16).unwrap()); + i += 2; + } + b => out.push(b), + } + i += 1; + } + String::from_utf8(out).unwrap() +} + +fn engine(stub: &Stub) -> ClickHouseEngine { + ClickHouseEngine::new(ClickHouseServer { + url: stub.url.clone(), + cgroup_agent_url: "http://10.0.0.2:9101".into(), + max_threads: 1, + statement_timeout: Duration::from_secs(5), + }) + .expect("engine") +} + +fn arm(plan: &str) -> ArmSpec { + ArmSpec { + name: "equivalent-only".into(), + raw_data_retained: true, + engine_options: json!({ "plan_file": fixture_path(plan) }), + } +} + +/// A workload whose queries are the plan's texts, with ids `q0`, `q1`, …, +/// evaluated every `interval_ms`, with data from `start_ms`. +fn workload_of(plan: &str, interval_ms: u64, start_ms: i64) -> WorkloadSpec { + let value = fixture_value(plan); + let texts: Vec = serde_json::from_value(value["query_texts"].clone()).unwrap(); + WorkloadSpec { + name: plan.into(), + queries: texts + .iter() + .enumerate() + .map(|(i, text)| WorkloadQuery { + id: format!("q{i}"), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: text.clone(), + }], + window_ms: 60_000, + evaluation_interval_ms: interval_ms, + }) + .collect(), + data: vec![data_record(start_ms)], + } +} + +fn data_record(start_ms: i64) -> DataRecord { + serde_json::from_value(json!({ + "metric": "latency", + "description": { + "labels": [], "series_num": 1, "start_ms": start_ms, "sample_interval_ms": 1000, + "duration_ms": 60000, + "value_distribution": {"kind": "uniform", "lower_bound": 0.0, "upper_bound": 1.0, "seed": 0}, + "counter": false, "seed": 1 + } + })) + .unwrap() +} + +fn labels(pairs: &[(&str, &str)]) -> Arc { + Arc::new( + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) +} + +fn statements(stub: &Stub) -> Vec { + stub.requests + .lock() + .unwrap() + .iter() + .map(|r| r.statement().to_string()) + .collect() +} + +#[test] +fn install_creates_a_database_and_the_translated_tables() { + // S1 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let installed = engine + .install( + &workload_of("rate_5m_stored", 60_000, 0), + &arm("rate_5m_stored"), + ) + .expect("installs"); + let requests = stub.requests.lock().unwrap().clone(); + // version, leftover check, CREATE DATABASE, src table, view table, view. + let created: Vec<&Request> = requests + .iter() + .filter(|r| r.statement().starts_with("CREATE")) + .collect(); + assert_eq!(created.len(), 4, "{requests:#?}"); + let database = created[0] + .statement() + .strip_prefix("CREATE DATABASE ") + .unwrap() + .to_string(); + assert!(database.starts_with("bench_")); + for request in &created[1..] { + assert_eq!(request.params.get("database"), Some(&database)); + assert_eq!( + request.params.get("wait_end_of_query").map(String::as_str), + Some("1") + ); + } + assert!(created[1] + .statement() + .starts_with("CREATE TABLE src_requests_total")); + assert!(created[2].statement().starts_with("CREATE TABLE v1")); + assert!(created[3] + .statement() + .starts_with("CREATE MATERIALIZED VIEW v1_mv")); + let summary = installed.plan_summary(); + assert_eq!( + summary.substitutions, + vec![ + "clickhouse.stored_rate.delta_sum_timestamp at node 1: a counter reset adds 0 \ + for the step across it, where PromQL adds the first value after the reset" + ] + ); + assert_eq!(summary.stored_outputs, vec!["v1"]); + assert_eq!(summary.node_kinds[1], "SummaryTumblingWindowChain"); + // S8: dropping the arm drops its database, synchronously. + drop(installed); + assert!(statements(&stub).contains(&format!("DROP DATABASE IF EXISTS {database} SYNC"))); + + let joint = engine + .install(&workload_of("joint_5m", 60_000, 0), &arm("joint_5m")) + .expect("installs"); + assert_eq!( + joint.plan_summary().sharing_lost, + vec!["node 1 for nodes 2, 3, 4"] + ); +} + +#[test] +fn install_maps_queries_by_index_and_text() { + // S2 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let mut workload = workload_of("sot_1h_stored_sum", 60_000, 0); + // Another language's text that equals the plan's is accepted. + workload.queries[0].texts.insert( + 0, + QueryText { + language: QueryLanguage::Sql, + text: "SELECT 1".into(), + }, + ); + assert!(engine.install(&workload, &arm("sot_1h_stored_sum")).is_ok()); + workload.queries[0].texts.remove(1); + let error = engine + .install(&workload, &arm("sot_1h_stored_sum")) + .err() + .expect("refused"); + assert!( + error + .0 + .contains("sum by (job) (sum_over_time(latency[1h]))") + && error.0.contains("SELECT 1"), + "{error}" + ); + let mut two = workload_of("sot_1h_stored_sum", 60_000, 0); + two.queries.push(two.queries[0].clone()); + assert!(engine + .install(&two, &arm("sot_1h_stored_sum")) + .err() + .expect("refused") + .0 + .contains("1 queries")); + // A stored step (60 s) that does not divide the evaluation interval. + let error = engine + .install( + &workload_of("sot_1h_stored_sum", 90_000, 0), + &arm("sot_1h_stored_sum"), + ) + .err() + .expect("refused"); + assert!(error.0.contains("60000 ms"), "{error}"); + // ... or the data start. + assert!(engine + .install( + &workload_of("sot_1h_stored_sum", 60_000, 30_000), + &arm("sot_1h_stored_sum") + ) + .is_err()); +} + +#[test] +fn install_refuses_missing_unreadable_and_untranslatable_plans() { + // S3 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let workload = workload_of("sot_1h_stored_sum", 60_000, 0); + let mut no_plan = arm("sot_1h_stored_sum"); + no_plan.engine_options = json!({}); + assert!(engine + .install(&workload, &no_plan) + .err() + .unwrap() + .0 + .contains("plan_file")); + let mut missing = arm("sot_1h_stored_sum"); + missing.engine_options = json!({"plan_file": "/nonexistent/plan.json"}); + assert!(engine + .install(&workload, &missing) + .err() + .unwrap() + .0 + .contains("/nonexistent")); + let mut value = fixture_value("sot_1h_stored_sum"); + payload_body(&mut value, 1)["summary_type"] = json!({"ExactAggregate": ["IRate", "IRate"]}); + let dir = std::env::temp_dir().join(format!("ch-server-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("irate.plan.json"); + std::fs::write(&path, value.to_string()).unwrap(); + let mut irate = arm("sot_1h_stored_sum"); + irate.engine_options = json!({ "plan_file": path }); + let error = engine.install(&workload, &irate).err().expect("refused"); + assert!( + error.0.contains("no translation for exact IRate"), + "{error}" + ); + // Nothing was created for a refused install. + assert!(!statements(&stub).iter().any(|s| s.starts_with("CREATE"))); +} + +#[test] +fn ingest_inserts_each_metric_in_chunk_order() { + // S4 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let mut installed = engine + .install( + &workload_of("max_5m_label_filter", 300_000, 0), + &arm("max_5m_label_filter"), + ) + .unwrap(); + let odd = "a\tb\nc\\d\"é"; + let a = labels(&[("__name__", "latency"), ("job", "j3")]); + let b = labels(&[("__name__", "latency"), ("instance", odd)]); + let other = labels(&[("__name__", "unread")]); + let chunk = SampleChunk { + end_ms: 2000, + samples: vec![ + Sample { + labels: Arc::clone(&a), + ts_ms: 1000, + value: 1.5, + }, + Sample { + labels: Arc::clone(&b), + ts_ms: 1000, + value: f64::NAN, + }, + Sample { + labels: other, + ts_ms: 1000, + value: 9.0, + }, + Sample { + labels: Arc::clone(&a), + ts_ms: 2000, + value: 2.0, + }, + ], + }; + let report = installed.ingest(&chunk).unwrap(); + assert_eq!(report.engine_reported_cpu_ns, None); + let requests = stub.requests.lock().unwrap().clone(); + let inserts: Vec<&Request> = requests + .iter() + .filter(|r| r.statement().starts_with("INSERT")) + .collect(); + assert_eq!( + inserts.len(), + 1, + "one insert for the one metric the plan reads" + ); + let insert = inserts[0]; + assert!( + insert.statement().starts_with( + "INSERT INTO src_latency (ts, value, `job`, `$promql_series_identity`) \ + SETTINGS input_format_null_as_default = 0, insert_null_as_default = 0 SELECT" + ), + "{}", + insert.statement() + ); + let escaped_identity = |l: &SeriesLabels| { + series_identity(l) + .replace('\\', "\\\\") + .replace('\t', "\\t") + .replace('\n', "\\n") + }; + assert_eq!( + insert.body, + format!( + "1000\t1.5\tj3\t{}\n1000\tnan\t\t{}\n2000\t2\tj3\t{}\n", + escaped_identity(&a), + escaped_identity(&b), + escaped_identity(&a) + ) + ); + assert_eq!(series_identity(&a), r#"{"__name__":"latency","job":"j3"}"#); + // A sample at 0 ms is refused (the window key needs ts > 0). + let zero = SampleChunk { + end_ms: 0, + samples: vec![Sample { + labels: a, + ts_ms: 0, + value: 1.0, + }], + }; + assert!(installed.ingest(&zero).is_err()); +} + +#[test] +fn evaluate_parses_series_and_values() { + // S5, S6, S6b + let identity = r#"{"__name__":"latency","instance":"a\tb","job":"j3"}"#; + let escaped = identity.replace('\\', "\\\\").replace('\t', "\\t"); + let answer = format!("job\t$promql_series_identity\tvalue\nj3\t{escaped}\tnan\n"); + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| { + ( + 200, + vec![( + "X-ClickHouse-Summary".to_string(), + r#"{"read_rows":"3","read_bytes":"48"}"#.to_string(), + )], + answer.clone(), + ) + }) + })]); + let engine = engine(&stub); + let mut installed = engine + .install( + &workload_of("max_5m_label_filter", 300_000, 0), + &arm("max_5m_label_filter"), + ) + .unwrap(); + let (answers, report) = installed.evaluate(600_000).unwrap(); + let series: SeriesLabels = serde_json::from_str(identity).unwrap(); + assert!(answers.get("q0", &series).unwrap().is_nan()); + assert_eq!(report.scanned_bytes, Some(48)); + assert_eq!(report.engine_reported_cpu_ns, None); + let query = stub + .requests + .lock() + .unwrap() + .iter() + .rev() + .find(|r| r.statement().contains("FORMAT TabSeparatedWithNames")) + .cloned() + .unwrap(); + assert_eq!( + query.params.get("param_t").map(String::as_str), + Some("600000") + ); + assert_eq!( + query.params.get("max_threads").map(String::as_str), + Some("1") + ); + assert!(query.params.get("database").unwrap().starts_with("bench_")); + + // Without an identity column: the label columns, empty ones left out; + // two rows of one series are an error. + for (body, expected) in [ + ( + "job\tvalue\nj3\t2\n\t5\n", + Ok(vec![(vec![("job", "j3")], 2.0), (vec![], 5.0)]), + ), + ("job\tvalue\nj3\t2\nj3\t5\n", Err("two rows")), + ] { + let body = body.to_string(); + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| (200, vec![], body.clone())) + })]); + let mut installed = engine_for(&stub); + match (installed.evaluate(600_000), expected) { + (Ok((answers, _)), Ok(rows)) => { + for (pairs, value) in rows { + let key: SeriesLabels = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + assert_eq!(answers.get("q0", &key), Some(value)); + } + } + (Err(error), Err(text)) => assert!(error.0.contains(text), "{error}"), + (got, want) => panic!("{got:?} vs {want:?}"), + } + } + + // S6: an error status; S6b: status 200 with an exception header. + for reply in [ + ( + 500, + vec![], + "Code: 395. DB::Exception: not a window end".to_string(), + ), + ( + 200, + vec![("X-ClickHouse-Exception-Code".to_string(), "395".to_string())], + "Code: 395. DB::Exception: not a window end".to_string(), + ), + ] { + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| reply.clone()) + })]); + let error = engine_for(&stub).evaluate(600_000).err().expect("an error"); + assert!(error.0.contains("not a window end"), "{error}"); + } +} + +fn engine_for(stub: &Stub) -> Box { + engine(stub) + .install( + &workload_of("max_5m_label_filter", 300_000, 0), + &arm("max_5m_label_filter"), + ) + .unwrap() +} + +#[test] +fn a_server_that_never_answers_times_out() { + // S6c: a listener that accepts and never answers. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let _held: Vec<_> = listener.incoming().take(4).collect(); + std::thread::sleep(Duration::from_secs(60)); + }); + let started = std::time::Instant::now(); + let result = ClickHouseEngine::new(ClickHouseServer { + url, + cgroup_agent_url: "http://10.0.0.2:9101".into(), + max_threads: 1, + statement_timeout: Duration::from_secs(1), + }); + assert!(result.is_err()); + assert!(started.elapsed() < Duration::from_secs(20)); +} + +#[test] +fn footprint_splits_view_and_raw_bytes() { + // S7 + let stub = start_stub(vec![Box::new(|request: &Request| { + request + .statement() + .contains("system.parts") + .then(|| (200, vec![], "src_latency\t100\nv1\t40\n".to_string())) + })]); + let engine = engine(&stub); + let installed = engine + .install( + &workload_of("sot_1h_stored_sum", 60_000, 0), + &arm("sot_1h_stored_sum"), + ) + .unwrap(); + let footprint = installed.footprint().unwrap(); + assert_eq!(footprint.view_bytes, 40); + assert_eq!(footprint.raw_retained_bytes, 100); +} + +#[test] +fn description_and_cpu_accounting() { + // S9 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let description = engine.description(); + assert_eq!(description.id, "clickhouse"); + assert_eq!(description.version, "26.8.39.13 (d5bbb64d)"); + assert_eq!(description.build_profile, "max_threads=1"); + assert_eq!( + engine.cpu_accounting(), + CpuAccounting::Cgroup(CgroupSource::Agent { + url: "http://10.0.0.2:9101".into() + }) + ); +} + +#[test] +fn leftover_databases_are_refused() { + // S10 + let stub = start_stub(vec![Box::new(|request: &Request| { + request + .statement() + .contains("system.databases") + .then(|| (200, vec![], "bench_1_0\nbench_1_1\n".to_string())) + })]); + let error = ClickHouseEngine::new(ClickHouseServer { + url: stub.url.clone(), + cgroup_agent_url: "http://10.0.0.2:9101".into(), + max_threads: 1, + statement_timeout: Duration::from_secs(5), + }) + .err() + .expect("refused"); + assert!(error.0.contains("bench_1_0, bench_1_1"), "{error}"); +} diff --git a/aqpbm-engines-external/tests/extbench_script.rs b/aqpbm-engines-external/tests/extbench_script.rs new file mode 100644 index 00000000..3335289e --- /dev/null +++ b/aqpbm-engines-external/tests/extbench_script.rs @@ -0,0 +1,44 @@ +//! `scripts/extbench-up.sh`: it parses, and it refuses to listen on an +//! address that is not private (the server has no password). + +use std::process::Command; + +fn script() -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../scripts/extbench-up.sh") +} + +#[test] +fn the_script_parses() { + let out = Command::new("bash") + .arg("-n") + .arg(script()) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[test] +fn only_private_listen_addresses_are_accepted() { + for (address, accepted) in [ + ("10.0.0.2", true), + ("172.16.4.1", true), + ("192.168.1.9", true), + ("127.0.0.1", true), + ("0.0.0.0", false), + ("8.8.8.8", false), + ("172.32.0.1", false), + ("10.0.0", false), + ("10.0.0.256", false), + ] { + let out = Command::new("bash") + .arg(script()) + .args(["check-listen-host", address]) + .output() + .unwrap(); + assert_eq!(out.status.success(), accepted, "{address}"); + } +} diff --git a/aqpbm-engines-external/tests/fixtures/plans/REV b/aqpbm-engines-external/tests/fixtures/plans/REV index 0d579c20..283d480c 100644 --- a/aqpbm-engines-external/tests/fixtures/plans/REV +++ b/aqpbm-engines-external/tests/fixtures/plans/REV @@ -18,6 +18,11 @@ Named fixtures (pretty-printed): joint_5m five 5m queries over latency and requests_total capabilities/clickhouse.equivalent-only.json (workloads/grid/joint__5m__1hz__60s.json) + probe_sum_10s sum_over_time(probe[10s]), every 10 s (the harness conformance probe) capabilities/clickhouse.equivalent-only.json + (workloads/probe/probe_sum_10s.json) + probe_count_10s count_over_time(probe[10s]), every 10 s capabilities/clickhouse.equivalent-only.json + (workloads/probe/probe_count_10s.json) + Public plans (`public/..plan.json`, compact): the workload `workloads//.json` planned with `capabilities/.json`. They are the eight RUN-1 showcase shapes (1 m windows, 1 Hz, every 10 s), `mixed-03`, diff --git a/aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json b/aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json new file mode 100644 index 00000000..a19b9e97 --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json @@ -0,0 +1,307 @@ +{ + "wire_version": 15, + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "NonASAP": { + "Scan": { + "source": { + "TimeSeries": { + "metric": "probe" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": "None", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "retention_ms": 10000 + }, + { + "id": 1, + "payload": { + "NonASAP": { + "TimeRange": { + "range": { + "secs": 10, + "nanos": 0 + }, + "kind": "range", + "child": 0 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + }, + { + "id": 2, + "payload": { + "NonASAP": { + "Aggregate": { + "reduction": "PerEntity", + "measures": [ + { + "kind": "count" + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null, + "child": 1 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + } + ], + "roots": [ + 2 + ], + "scalar_roots": [] + }, + "query_texts": [ + "count_over_time(probe[10s])" + ] +} \ No newline at end of file diff --git a/aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json b/aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json new file mode 100644 index 00000000..b7574ea2 --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json @@ -0,0 +1,308 @@ +{ + "wire_version": 15, + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "NonASAP": { + "Scan": { + "source": { + "TimeSeries": { + "metric": "probe" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": "None", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "retention_ms": 10000 + }, + { + "id": 1, + "payload": { + "NonASAP": { + "TimeRange": { + "range": { + "secs": 10, + "nanos": 0 + }, + "kind": "range", + "child": 0 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + }, + { + "id": 2, + "payload": { + "NonASAP": { + "Aggregate": { + "reduction": "PerEntity", + "measures": [ + { + "kind": "sum", + "col": null + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null, + "child": 1 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + } + ], + "roots": [ + 2 + ], + "scalar_roots": [] + }, + "query_texts": [ + "sum_over_time(probe[10s])" + ] +} \ No newline at end of file diff --git a/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json new file mode 100644 index 00000000..d3b3eee6 --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json @@ -0,0 +1,36 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "count_over_time(probe[10s])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": {}, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 0.1 + } + }, + "sources": { + "probe": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 10000 + } + } + } +} \ No newline at end of file diff --git a/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json new file mode 100644 index 00000000..cfda417c --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json @@ -0,0 +1,36 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "sum_over_time(probe[10s])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": {}, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 0.1 + } + }, + "sources": { + "probe": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 10000 + } + } + } +} \ No newline at end of file From f1468e050f48ff7a7bc542edf36eea8c9644a669 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 17 Nov 2026 14:20:49 +0000 Subject: [PATCH 2/6] engines-external, cli: the ClickHouse adapter and the clickhouse lane - ClickHouseEngine (EngineUnderTest over HTTP): each install translates its plan file into a fresh database; one insert per source table per chunk; every query root at t with param_t, max_threads and wait_end_of_query; answers keyed by the workload query ids, series by the decoded series identity or by the label columns; footprint from system.parts; CPU from the server's cgroup through the agent (the server reports no per-query CPU in its summary header, so the engine counter is None). - The translator also returns each metric's source table and label columns and the stored tables. - approxbench bench-external run --lane clickhouse --clickhouse-url URL --cgroup-agent-url URL [--clickhouse-max-threads N]. - scripts/extbench-up.sh starts the server in a pinned container from a checked binary, with settings as command-line overrides (without a main config file the server reads no config.d), on a private address only, with the cgroup agent beside it and versions.txt. --- aqpbm-cli/src/bench_external_cmd.rs | 34 +- aqpbm-engines-external/src/clickhouse.rs | 53 +- .../src/clickhouse_server.rs | 607 ++++++++++++++++++ aqpbm-engines-external/src/lib.rs | 1 + scripts/extbench-up.sh | 111 ++++ 5 files changed, 793 insertions(+), 13 deletions(-) create mode 100644 aqpbm-engines-external/src/clickhouse_server.rs create mode 100755 scripts/extbench-up.sh diff --git a/aqpbm-cli/src/bench_external_cmd.rs b/aqpbm-cli/src/bench_external_cmd.rs index 3a00905d..a3994969 100644 --- a/aqpbm-cli/src/bench_external_cmd.rs +++ b/aqpbm-cli/src/bench_external_cmd.rs @@ -9,6 +9,7 @@ use std::net::TcpListener; use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; +use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; use aqpbm_core::harness::bench_external::{report, run_lane, write_report, LaneSetup}; use aqpbm_core::harness::cgroup::{agent_in_served_group, serve_agent}; use aqpbm_core::harness::engine::EngineUnderTest; @@ -20,7 +21,7 @@ use aqpbm_core::harness::rundir::RunDir; use clap::{Parser, Subcommand}; /// The lanes this binary can run. -pub const LANES: [&str; 1] = ["fake"]; +pub const LANES: [&str; 2] = ["fake", "clickhouse"]; #[derive(Parser, Debug)] pub struct BenchExternalArgs { @@ -41,9 +42,23 @@ pub struct RunArgs { /// The run directory: workloads/, plans//, results/. #[arg(long)] pub run_dir: PathBuf, - /// The lane: `fake` (the harness's own fake engine). + /// The lane: `fake` (the harness's own fake engine) or `clickhouse`. #[arg(long)] pub lane: String, + /// The ClickHouse server's HTTP interface, e.g. `http://10.0.0.1:8123` + /// (lane `clickhouse`). + #[arg(long)] + pub clickhouse_url: Option, + /// The cgroup agent beside the server, e.g. `http://10.0.0.1:9101` + /// (lane `clickhouse`). + #[arg(long)] + pub cgroup_agent_url: Option, + /// `max_threads` of every ClickHouse statement; 1 for the main figures. + #[arg(long, default_value_t = 1)] + pub clickhouse_max_threads: u32, + /// How long one ClickHouse statement may run, in seconds. + #[arg(long, default_value_t = 600)] + pub clickhouse_statement_timeout_s: u64, /// Only these variants (repeatable); every variant under plans/ by default. #[arg(long = "variant")] pub variants: Vec, @@ -94,6 +109,21 @@ pub fn run(args: BenchExternalArgs) -> Result<()> { fn run_lane_cmd(args: RunArgs) -> Result<()> { let engine: Box = match args.lane.as_str() { "fake" => Box::new(FakeEngine::default()), + "clickhouse" => { + let (Some(url), Some(cgroup_agent_url)) = + (args.clickhouse_url.clone(), args.cgroup_agent_url.clone()) + else { + bail!("lane clickhouse needs --clickhouse-url and --cgroup-agent-url"); + }; + Box::new(ClickHouseEngine::new(ClickHouseServer { + url, + cgroup_agent_url, + max_threads: args.clickhouse_max_threads, + statement_timeout: std::time::Duration::from_secs( + args.clickhouse_statement_timeout_s, + ), + })?) + } other => bail!("unknown lane '{other}'; known lanes: {}", LANES.join(", ")), }; let run_dir = RunDir::open(&args.run_dir)?; diff --git a/aqpbm-engines-external/src/clickhouse.rs b/aqpbm-engines-external/src/clickhouse.rs index 748ec43b..9a864e4c 100644 --- a/aqpbm-engines-external/src/clickhouse.rs +++ b/aqpbm-engines-external/src/clickhouse.rs @@ -49,7 +49,7 @@ use crate::{NodeId, SharingLost, Substitution, TranslationError}; /// The plan's series identity column, a label column like any other. It has /// one value per series, so it is a plain `String`: `LowCardinality` pays /// off only below about ten thousand distinct values. -const SERIES_IDENTITY: &str = "$promql_series_identity"; +pub const SERIES_IDENTITY: &str = "$promql_series_identity"; /// The settings every `INSERT` into a source table uses: a NULL value is an /// error instead of being stored as 0 (PromQL samples are never NULL), in @@ -62,6 +62,11 @@ pub const INSERT_SETTINGS: &str = "input_format_null_as_default = 0, insert_null pub struct ClickHouseDeployment { /// `CREATE TABLE` statements: one per metric, then one per stored chain. pub tables: Vec, + /// The source table of each metric, in the order of `tables`: where its + /// samples are inserted. + pub sources: Vec, + /// The tables of the stored chains (`v`), in the order of `tables`. + pub stored_tables: Vec, /// `CREATE MATERIALIZED VIEW` statements, one per stored chain. pub materialized_views: Vec, /// One `SELECT` per query root, in root order, with the parameter `{t:Int64}`. @@ -77,6 +82,16 @@ pub struct ClickHouseDeployment { pub sharing_lost: Vec, } +/// A metric's source table: samples of `metric` (their `__name__`) are +/// inserted into `table`, with one column per label of `label_columns`, in +/// that order, the series identity last. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceTable { + pub metric: String, + pub table: String, + pub label_columns: Vec, +} + /// A table's retention: rows whose time is older than `retention_ms` before /// the newest evaluation are no longer read. #[derive(Debug, Clone, PartialEq, Eq)] @@ -136,12 +151,15 @@ pub fn translate(plan: &PlanFile) -> Result, Vec) = + source_tables(plan).into_iter().unzip(); + let mut stored_tables = Vec::new(); let mut table_retention = source_retention(plan); let mut materialized_views = Vec::new(); let mut substitutions = Vec::new(); for chain in translator.chains.values() { tables.push(chain.table()); + stored_tables.push(chain.table_name()); materialized_views.push(chain.view()); if let Some(retention_ms) = chain.retention_ms { table_retention.push(TableRetention { @@ -161,6 +179,8 @@ pub fn translate(plan: &PlanFile) -> Result String { /// primary key; a `minmax` index on `ts` skips the granules outside a time /// range. A metric only stored chains read keeps no rows (`ENGINE = Null`): /// its materialized views still see every insert. -fn source_tables(plan: &PlanFile) -> Vec { +fn source_tables(plan: &PlanFile) -> Vec<(SourceTable, String)> { struct SourceColumns<'a> { metric: &'a str, labels: Vec<&'a str>, @@ -421,10 +441,18 @@ fn source_tables(plan: &PlanFile) -> Vec { .map(|label| format!(", {} {}", quoted(label), label_type(label))) .collect(); let table = source_table(source.metric); + let described = SourceTable { + metric: source.metric.to_string(), + table: table.clone(), + label_columns: source.labels.iter().map(|l| l.to_string()).collect(), + }; if !source.keeps_rows { - return format!( - "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}) \ - ENGINE = Null" + return ( + described, + format!( + "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}) \ + ENGINE = Null" + ), ); } let sorting_key: Vec = source @@ -434,11 +462,14 @@ fn source_tables(plan: &PlanFile) -> Vec { .map(|label| quoted(label)) .chain(["ts".to_string()]) .collect(); - format!( - "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}, \ - INDEX ts_minmax ts TYPE minmax GRANULARITY 1) \ - ENGINE = MergeTree ORDER BY ({})", - sorting_key.join(", ") + ( + described, + format!( + "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}, \ + INDEX ts_minmax ts TYPE minmax GRANULARITY 1) \ + ENGINE = MergeTree ORDER BY ({})", + sorting_key.join(", ") + ), ) }) .collect() diff --git a/aqpbm-engines-external/src/clickhouse_server.rs b/aqpbm-engines-external/src/clickhouse_server.rs new file mode 100644 index 00000000..23139755 --- /dev/null +++ b/aqpbm-engines-external/src/clickhouse_server.rs @@ -0,0 +1,607 @@ +//! A ClickHouse server as an engine under test: each arm is a plan file +//! translated by [`crate::clickhouse::translate`] and installed in a fresh +//! database of the server over HTTP. +//! +//! Example: an arm whose `engine_options` are `{"plan_file": "q4c.plan.json"}` +//! installs `src_latency` in database `bench_4711_0`. Each refresh chunk is +//! one `INSERT` per source table, and each evaluation at `t` runs the plan's +//! queries with `param_t=`. +//! +//! Series: a sample's series identity is its full label map as compact +//! JSON with sorted keys, e.g. `{"__name__":"latency","job":"a"}`. A +//! per-series answer is keyed by that map (`__name__` kept, as the harness +//! keys answers). An aggregated answer is keyed by its label columns, with +//! empty values left out, as PromQL leaves out an empty label. +//! +//! CPU: the server runs in its own cgroup, read through the cgroup agent. +//! The server's HTTP summary header has no CPU counter, so +//! [`StepReport::engine_reported_cpu_ns`] is `None`; `scanned_bytes` is the +//! summed `read_bytes` of the step's statements. + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use aqpbm_core::harness::cgroup::CgroupSource; +use aqpbm_core::harness::engine::{ + Answers, ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, Footprint, + InstalledArm, PlanSummary, SampleChunk, SeriesLabels, StepReport, WorkloadSpec, +}; + +use crate::clickhouse::{translate, SourceTable, INSERT_SETTINGS, SERIES_IDENTITY}; +use crate::plan_file::{PlanFile, PlanOperator}; + +/// How to reach the server and how to run its queries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClickHouseServer { + /// The HTTP interface, e.g. `http://10.0.0.1:8123`. + pub url: String, + /// The cgroup agent beside the server, e.g. `http://10.0.0.1:9101`. + pub cgroup_agent_url: String, + /// `max_threads` of every statement (1 for the main figures). + pub max_threads: u32, + /// How long one statement may take before it fails, e.g. 600 s. + pub statement_timeout: Duration, +} + +/// A ClickHouse server; see the module docs. +pub struct ClickHouseEngine { + server: ClickHouseServer, + http: Http, + description: EngineDescription, + /// Each install's database is `_`. + database_prefix: String, + installs: AtomicU64, +} + +impl ClickHouseEngine { + /// Connects to the server and reads its version and git hash. A + /// database of an earlier run (`bench_*`) is refused: dropping it, or its + /// background merges, would be charged to this run's measurements. + pub fn new(server: ClickHouseServer) -> Result { + let http = Http::new(&server.url, server.statement_timeout); + let (leftovers, _) = http.run( + "SELECT name FROM system.databases WHERE startsWith(name, 'bench_') FORMAT TabSeparated", + &[], + None, + )?; + if !leftovers.trim().is_empty() { + return Err(EngineError(format!( + "the server holds databases of an earlier run: {}; start it from an empty data dir", + leftovers.trim().replace('\n', ", ") + ))); + } + let (text, _) = http.run( + "SELECT version(), (SELECT value FROM system.build_options \ + WHERE name = 'VERSION_GITHASH') FORMAT TabSeparated", + &[], + None, + )?; + let (version, git_hash) = text + .trim_end() + .split_once('\t') + .ok_or_else(|| EngineError(format!("unexpected version answer '{text}'")))?; + let description = EngineDescription { + id: "clickhouse".into(), + version: format!("{version} ({git_hash})"), + version_dirty: false, + build_profile: format!("max_threads={}", server.max_threads), + }; + Ok(ClickHouseEngine { + database_prefix: format!("bench_{}", std::process::id()), + server, + http, + description, + installs: AtomicU64::new(0), + }) + } +} + +impl EngineUnderTest for ClickHouseEngine { + fn description(&self) -> EngineDescription { + self.description.clone() + } + + fn cpu_accounting(&self) -> CpuAccounting { + CpuAccounting::Cgroup(CgroupSource::Agent { + url: self.server.cgroup_agent_url.clone(), + }) + } + + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let path = arm + .engine_options + .get("plan_file") + .and_then(|p| p.as_str()) + .ok_or_else(|| EngineError("engine_options has no plan_file".into()))?; + let text = std::fs::read_to_string(path) + .map_err(|error| EngineError(format!("plan file {path}: {error}")))?; + let plan = PlanFile::from_json(&text) + .map_err(|error| EngineError(format!("plan file {path}: {error}")))?; + let query_ids = query_ids_of(&plan, workload)?; + let deployment = + translate(&plan).map_err(|error| EngineError(format!("plan file {path}: {error}")))?; + check_evaluation_times(workload, &deployment.query_time_steps_ms)?; + if let Some(source) = deployment.sources.iter().find(|source| { + source + .label_columns + .iter() + .any(|label| label == "ts" || label == "value") + }) { + return Err(EngineError(format!( + "metric {} has a label named ts or value, the names of its sample columns", + source.metric + ))); + } + + let database = format!( + "{}_{}", + self.database_prefix, + self.installs.fetch_add(1, Ordering::Relaxed) + ); + self.http + .run(&format!("CREATE DATABASE {database}"), &[], None)?; + // From here on, dropping the arm drops the database. + let installed = ClickHouseArm { + http: self.http.clone(), + database, + max_threads: self.server.max_threads, + sources: deployment.sources.clone(), + queries: query_ids + .into_iter() + .zip(deployment.queries.clone()) + .collect(), + stored_tables: deployment.stored_tables.clone(), + summary: plan_summary(&plan, &deployment), + }; + // TTL statements are never applied: the harness replays data, and a + // TTL deletes by wall-clock time. + for statement in deployment + .tables + .iter() + .chain(&deployment.materialized_views) + { + installed.run(statement, &[], None)?; + } + Ok(Box::new(installed)) + } +} + +/// Every evaluation time is a multiple of the evaluation interval from an +/// aligned data start ([`aqpbm_core::harness::arm_run::ArmSchedule`]), so a query +/// that reads stored windows answers right only when its step divides the +/// interval and the data start. Refused at install, not at the first +/// evaluation. +fn check_evaluation_times(workload: &WorkloadSpec, steps_ms: &[u64]) -> Result<(), EngineError> { + let data_start_ms = workload + .data + .iter() + .map(|record| record.description.start_ms) + .min() + .unwrap_or(0); + for (query, &step_ms) in workload.queries.iter().zip(steps_ms) { + let step = step_ms as i64; + if (query.evaluation_interval_ms as i64) % step != 0 || data_start_ms % step != 0 { + return Err(EngineError(format!( + "query {} reads stored windows of {step_ms} ms, which do not divide its \ + evaluation interval ({} ms) and the data start ({data_start_ms} ms)", + query.id, query.evaluation_interval_ms + ))); + } + } + Ok(()) +} + +/// The workload query id of each plan root: root i answers query i, whose +/// texts (in any language) must include the plan's text of root i. +fn query_ids_of(plan: &PlanFile, workload: &WorkloadSpec) -> Result, EngineError> { + if plan.query_texts.len() != workload.queries.len() { + return Err(EngineError(format!( + "the plan has {} queries but workload {} has {}", + plan.query_texts.len(), + workload.name, + workload.queries.len() + ))); + } + plan.query_texts + .iter() + .zip(&workload.queries) + .map(|(plan_text, query)| { + if query.texts.iter().any(|text| &text.text == plan_text) { + Ok(query.id.clone()) + } else { + let texts: Vec<&str> = query.texts.iter().map(|t| t.text.as_str()).collect(); + Err(EngineError(format!( + "query {}: the plan answers '{plan_text}', but the workload's texts are {texts:?}", + query.id + ))) + } + }) + .collect() +} + +fn plan_summary( + plan: &PlanFile, + deployment: &crate::clickhouse::ClickHouseDeployment, +) -> PlanSummary { + PlanSummary { + text: deployment.script(), + node_kinds: plan + .nodes + .iter() + .map(|node| operator_kind(&node.operator)) + .collect(), + stored_outputs: deployment.stored_tables.clone(), + substitutions: deployment + .substitutions + .iter() + .map(|s| format!("{} at node {}: {}", s.rule, s.node, s.guarantee_difference)) + .collect(), + sharing_lost: deployment + .sharing_lost + .iter() + .map(|lost| { + let readers: Vec = lost.readers.iter().map(|r| r.to_string()).collect(); + format!("node {} for nodes {}", lost.node, readers.join(", ")) + }) + .collect(), + } +} + +/// The planner's operator name of `operator`, e.g. `SummaryAgg`. +fn operator_kind(operator: &PlanOperator) -> String { + match operator { + PlanOperator::Scan { .. } => "Scan", + PlanOperator::TimeRange { .. } => "TimeRange", + PlanOperator::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain", + PlanOperator::SummaryAgg { .. } => "SummaryAgg", + PlanOperator::SummaryMerge { .. } => "SummaryMerge", + PlanOperator::FinalizeExactAccumulator { .. } => "FinalizeExactAccumulator", + PlanOperator::SummaryEstimate { .. } => "SummaryEstimate", + PlanOperator::Aggregate { .. } => "Aggregate", + PlanOperator::Project { .. } => "Project", + PlanOperator::BinaryOp { .. } => "BinaryOp", + PlanOperator::Other { kind } => kind, + } + .to_string() +} + +/// One installed arm: its database and the queries it answers. +struct ClickHouseArm { + http: Http, + database: String, + max_threads: u32, + sources: Vec, + /// Per plan root: the workload query id and its SQL. + queries: Vec<(String, String)>, + stored_tables: Vec, + summary: PlanSummary, +} + +impl ClickHouseArm { + /// Runs `sql` in the arm's database with its `max_threads`. + fn run( + &self, + sql: &str, + extra: &[(&str, &str)], + insert_rows: Option<&str>, + ) -> Result<(String, u64), EngineError> { + let threads = self.max_threads.to_string(); + let mut params = vec![ + ("database", self.database.as_str()), + ("max_threads", threads.as_str()), + ]; + params.extend_from_slice(extra); + self.http.run(sql, ¶ms, insert_rows) + } +} + +impl Drop for ClickHouseArm { + fn drop(&mut self) { + // Best effort: a server that is gone keeps nothing anyway. + let _ = self.http.run( + &format!("DROP DATABASE IF EXISTS {} SYNC", self.database), + &[], + None, + ); + } +} + +impl InstalledArm for ClickHouseArm { + fn plan_summary(&self) -> PlanSummary { + self.summary.clone() + } + + fn ingest(&mut self, chunk: &SampleChunk) -> Result { + let mut scanned_bytes = 0; + for source in &self.sources { + let mut rows = String::new(); + for sample in &chunk.samples { + if sample.labels.get("__name__") != Some(&source.metric) { + continue; + } + if sample.ts_ms <= 0 { + return Err(EngineError(format!( + "a sample at {} ms: timestamps must be after 0", + sample.ts_ms + ))); + } + rows.push_str(&sample.ts_ms.to_string()); + rows.push('\t'); + rows.push_str(&float_text(sample.value)); + for label in &source.label_columns { + rows.push('\t'); + let value = if label == SERIES_IDENTITY { + series_identity(&sample.labels) + } else { + sample.labels.get(label).cloned().unwrap_or_default() + }; + push_tsv_escaped(&mut rows, &value); + } + rows.push('\n'); + } + if rows.is_empty() { + continue; + } + let (_, read_bytes) = self.run(&insert_statement(source), &[], Some(&rows))?; + scanned_bytes += read_bytes; + } + Ok(StepReport { + scanned_bytes: Some(scanned_bytes), + engine_reported_cpu_ns: None, + }) + } + + fn evaluate(&mut self, at_ms: i64) -> Result<(Answers, StepReport), EngineError> { + let t = at_ms.to_string(); + let mut answers = BTreeMap::new(); + let mut scanned_bytes = 0; + for (id, sql) in &self.queries { + let (text, read_bytes) = self.run( + &format!("{sql} FORMAT TabSeparatedWithNames"), + &[("param_t", &t)], + None, + )?; + scanned_bytes += read_bytes; + answers.insert( + id.clone(), + parse_answer(&text).map_err(|e| EngineError(format!("query {id}: {e}")))?, + ); + } + Ok(( + Answers(answers), + StepReport { + scanned_bytes: Some(scanned_bytes), + engine_reported_cpu_ns: None, + }, + )) + } + + fn footprint(&self) -> Result { + let (text, _) = self.run( + &format!( + "SELECT table, sum(bytes_on_disk) FROM system.parts \ + WHERE database = '{}' AND active GROUP BY table FORMAT TabSeparated", + self.database + ), + &[], + None, + )?; + let mut footprint = Footprint::default(); + for line in text.lines() { + let (table, bytes) = line + .split_once('\t') + .ok_or_else(|| EngineError(format!("unexpected parts row '{line}'")))?; + let bytes: u64 = bytes + .parse() + .map_err(|_| EngineError(format!("unexpected parts row '{line}'")))?; + if self.stored_tables.iter().any(|t| t == table) { + footprint.view_bytes += bytes; + } else if self.sources.iter().any(|s| s.table == table) { + footprint.raw_retained_bytes += bytes; + } + } + Ok(footprint) + } +} + +/// `INSERT … SELECT … FROM input(…) FORMAT TabSeparated` into `source`: rows +/// of `ts_ms`, `value`, then one column per label column. +fn insert_statement(source: &SourceTable) -> String { + let inputs: Vec = (0..source.label_columns.len()) + .map(|i| format!("c{i} String")) + .collect(); + let selected: Vec = (0..source.label_columns.len()) + .map(|i| format!("c{i}")) + .collect(); + let columns: Vec = source + .label_columns + .iter() + .map(|label| format!("`{}`", label.replace('`', "\\`"))) + .collect(); + format!( + "INSERT INTO {table} (ts, value, {columns}) SETTINGS {INSERT_SETTINGS} \ + SELECT fromUnixTimestamp64Milli(ts_ms), value, {selected} \ + FROM input('ts_ms Int64, value Float64, {inputs}') FORMAT TabSeparated", + table = source.table, + columns = columns.join(", "), + selected = selected.join(", "), + inputs = inputs.join(", "), + ) +} + +/// A series' identity: its label map as compact JSON, keys sorted. +pub fn series_identity(labels: &SeriesLabels) -> String { + serde_json::to_string(labels).expect("a string map serializes") +} + +/// `value` as TabSeparated reads it: `nan`, `inf`, `-inf`, or the shortest +/// decimal that reads back exactly. +fn float_text(value: f64) -> String { + if value.is_nan() { + "nan".into() + } else { + value.to_string() + } +} + +fn push_tsv_escaped(out: &mut String, value: &str) { + for c in value.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + c => out.push(c), + } + } +} + +fn tsv_unescaped(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('t') => out.push('\t'), + Some('n') => out.push('\n'), + Some('0') => out.push('\0'), + Some(other) => out.push(other), + None => out.push('\\'), + } + } + out +} + +/// The rows of one query answer (`TabSeparatedWithNames`, the value +/// column last) as series and values. +fn parse_answer(text: &str) -> Result, String> { + let mut lines = text.lines(); + let Some(header) = lines.next() else { + return Ok(BTreeMap::new()); + }; + let names: Vec = header.split('\t').map(tsv_unescaped).collect(); + let mut series = BTreeMap::new(); + for line in lines { + let fields: Vec = line.split('\t').map(tsv_unescaped).collect(); + if fields.len() != names.len() { + return Err(format!("row '{line}' does not match the header '{header}'")); + } + let (value, labels) = fields.split_last().expect("at least the value"); + let identity = names.iter().position(|name| name == SERIES_IDENTITY); + let key: SeriesLabels = match identity { + Some(at) => serde_json::from_str(&labels[at]) + .map_err(|e| format!("series identity '{}': {e}", labels[at]))?, + None => names + .iter() + .zip(labels) + .filter(|(_, value)| !value.is_empty()) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + }; + let value = parse_float(value).ok_or_else(|| format!("value '{value}' is not a number"))?; + if series.insert(key.clone(), value).is_some() { + return Err(format!("two rows for the series {key:?}")); + } + } + Ok(series) +} + +fn parse_float(text: &str) -> Option { + match text { + "nan" | "-nan" => Some(f64::NAN), + "inf" | "+inf" => Some(f64::INFINITY), + "-inf" => Some(f64::NEG_INFINITY), + _ => text.parse().ok(), + } +} + +/// The server's HTTP interface. A statement is the `query` parameter; an +/// insert's rows are the body. +#[derive(Clone)] +struct Http { + agent: ureq::Agent, + url: String, + /// `max_execution_time` of every statement, in whole seconds. + max_execution_seconds: String, +} + +/// How long to wait to connect. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +impl Http { + fn new(url: &str, statement_timeout: Duration) -> Http { + Http { + // The client waits a little longer than the server lets a + // statement run, so the server's own timeout error arrives. + agent: ureq::AgentBuilder::new() + .timeout_connect(CONNECT_TIMEOUT) + .timeout(statement_timeout + Duration::from_secs(5)) + .build(), + url: url.trim_end_matches('/').to_string(), + max_execution_seconds: statement_timeout.as_secs().max(1).to_string(), + } + } + + /// Runs `sql` with the URL parameters `params`; its output and the + /// `read_bytes` of its summary header. The statement is the request + /// body, except for an insert, whose body is its rows. + fn run( + &self, + sql: &str, + params: &[(&str, &str)], + insert_rows: Option<&str>, + ) -> Result<(String, u64), EngineError> { + // `wait_end_of_query`: the status and the summary header come after + // the statement ends, so an error raised mid-answer is a status + // error and `read_bytes` is complete. + let mut request = self + .agent + .post(&format!("{}/", self.url)) + .query("wait_end_of_query", "1") + .query("max_execution_time", &self.max_execution_seconds); + for (name, value) in params { + request = request.query(name, value); + } + let sent = match insert_rows { + Some(rows) => request.query("query", sql).send_string(rows), + None => request.send_string(sql), + }; + match sent { + Ok(response) => { + if let Some(code) = response.header("X-ClickHouse-Exception-Code") { + let code = code.to_string(); + let text = response.into_string().unwrap_or_default(); + return Err(EngineError(format!( + "ClickHouse exception {code}: {}", + text.trim_end() + ))); + } + let read_bytes = response + .header("X-ClickHouse-Summary") + .and_then(|summary| serde_json::from_str::(summary).ok()) + .and_then(|summary| summary.get("read_bytes")?.as_str()?.parse().ok()) + .unwrap_or(0); + // `into_string` stops at 10 MB; an answer may be larger. + let mut text = String::new(); + std::io::Read::read_to_string(&mut response.into_reader(), &mut text) + .map_err(|error| EngineError(format!("reading the answer: {error}")))?; + Ok((text, read_bytes)) + } + Err(ureq::Error::Status(code, response)) => { + let text = response.into_string().unwrap_or_default(); + Err(EngineError(format!( + "ClickHouse answered {code}: {}", + text.trim_end() + ))) + } + Err(error) => Err(EngineError(format!("ClickHouse at {}: {error}", self.url))), + } + } +} diff --git a/aqpbm-engines-external/src/lib.rs b/aqpbm-engines-external/src/lib.rs index d0dada66..0a520a57 100644 --- a/aqpbm-engines-external/src/lib.rs +++ b/aqpbm-engines-external/src/lib.rs @@ -11,6 +11,7 @@ //! what each engine can run; their README gives the evidence per entry. pub mod clickhouse; +pub mod clickhouse_server; pub mod plan_file; /// A node's position in the plan file's DAG. diff --git a/scripts/extbench-up.sh b/scripts/extbench-up.sh new file mode 100755 index 00000000..e17c0f11 --- /dev/null +++ b/scripts/extbench-up.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Start (or stop) one measured server for `approxbench bench-external` on +# this host: a docker container pinned to given cores, with the cgroup +# agent beside it. +# +# scripts/extbench-up.sh clickhouse up [] +# scripts/extbench-up.sh clickhouse down [] +# scripts/extbench-up.sh check-listen-host
+# +# Example: LISTEN_HOST=10.0.0.2 scripts/extbench-up.sh clickhouse up /data/run-1/clickhouse 2,22 0 18123 19101 +# +# Environment: +# LISTEN_HOST address the server and the agent listen on (required): +# a private (10/8, 172.16/12, 192.168/16) or loopback +# address, never 0.0.0.0 or a public one: the server +# has no password +# CLICKHOUSE_BINARY path of the ClickHouse binary to bind-mount (required) +# CLICKHOUSE_SHA256 its expected sha256 (required); a mismatch refuses +# BASE_IMAGE the image the binary runs in, pinned by digest (required) +# APPROXBENCH path of the approxbench binary, for `cgroup-agent` (required) +# +# Every start begins from an empty data dir. The script only deletes a data +# dir it created (it holds the marker file .extbench-data-dir). The +# container runs as the invoking user, with no CPU quota (--cpus unset) and +# no memory limit. ClickHouse runs on its embedded configuration: without a +# main config file it reads no config.d, so the few settings changed here +# are `--key=value` overrides. +set -euo pipefail + +marker=.extbench-data-dir + +# Exit 0 when $1 is a private or loopback IPv4 address. +private_address() { + local a b c d + IFS=. read -r a b c d <<<"$1" || return 1 + for octet in "$a" "$b" "$c" "$d"; do + [[ "$octet" =~ ^[0-9]+$ ]] && [ "$octet" -le 255 ] || return 1 + done + [ "$a" = 10 ] && return 0 + [ "$a" = 127 ] && return 0 + [ "$a" = 172 ] && [ "$b" -ge 16 ] && [ "$b" -le 31 ] && return 0 + [ "$a" = 192 ] && [ "$b" = 168 ] && return 0 + return 1 +} + +if [ "${1:-}" = check-listen-host ]; then + private_address "${2:?address}" || { echo "refused: ${2} is not a private address" >&2; exit 1; } + exit 0 +fi + +engine=${1:?engine}; action=${2:?up|down}; data_dir=${3:?data dir} +[ "$engine" = clickhouse ] || { echo "unknown engine $engine" >&2; exit 2; } + +if [ "$action" = down ]; then + name=${4:-extbench-clickhouse} + [ -f "$data_dir/agent.pid" ] && kill "$(cat "$data_dir/agent.pid")" 2>/dev/null || true + docker rm -f "$name" >/dev/null 2>&1 || true + exit 0 +fi + +cores=${4:?cores}; numa=${5:?numa node}; http_port=${6:?http port}; agent_port=${7:?agent port} +name=${8:-extbench-clickhouse} +: "${LISTEN_HOST:?set LISTEN_HOST to a private address}" "${CLICKHOUSE_BINARY:?}" \ + "${CLICKHOUSE_SHA256:?}" "${BASE_IMAGE:?}" "${APPROXBENCH:?}" +private_address "$LISTEN_HOST" || { echo "refused: LISTEN_HOST $LISTEN_HOST is not a private address" >&2; exit 1; } +actual=$(sha256sum "$CLICKHOUSE_BINARY" | cut -d' ' -f1) +[ "$actual" = "$CLICKHOUSE_SHA256" ] || { echo "binary sha256 $actual != $CLICKHOUSE_SHA256" >&2; exit 1; } + +if [ -e "$data_dir" ]; then + [ -f "$data_dir/$marker" ] || { echo "refused: $data_dir exists and was not made by this script" >&2; exit 1; } + [ -f "$data_dir/agent.pid" ] && kill "$(cat "$data_dir/agent.pid")" 2>/dev/null || true + docker rm -f "$name" >/dev/null 2>&1 || true + rm -rf "$data_dir" +fi +mkdir -p "$data_dir/db"; touch "$data_dir/$marker" + +docker rm -f "$name" >/dev/null 2>&1 || true +docker run -d --name "$name" --network host \ + --cpuset-cpus "$cores" --cpuset-mems "$numa" --user "$(id -u):$(id -g)" \ + -v "$CLICKHOUSE_BINARY:/usr/bin/clickhouse:ro" -v "$data_dir/db:/var/lib/clickhouse" \ + -w /var/lib/clickhouse \ + "$BASE_IMAGE" /usr/bin/clickhouse server -- --path=/var/lib/clickhouse/ \ + --http_port="$http_port" --tcp_port=0 --mysql_port=0 --postgresql_port=0 --interserver_http_port=0 \ + --listen_host="$LISTEN_HOST" --logger.level=warning --display_name="$name" >/dev/null +for _ in $(seq 1 60); do + curl -sf "http://$LISTEN_HOST:$http_port/ping" >/dev/null && break; sleep 1 +done +curl -sf "http://$LISTEN_HOST:$http_port/ping" >/dev/null || { echo "the server did not start" >&2; docker logs "$name" >&2; exit 1; } + +id=$(docker inspect -f '{{.Id}}' "$name") +group="/sys/fs/cgroup/system.slice/docker-$id.scope" +[ -d "$group" ] || { echo "no cgroup at $group (is docker on the systemd cgroup driver?)" >&2; exit 1; } +nohup "$APPROXBENCH" cgroup-agent --cgroup-dir "$group" --listen "$LISTEN_HOST:$agent_port" \ + > "$data_dir/agent.log" 2>&1 & echo $! > "$data_dir/agent.pid" + +query() { curl -sf "http://$LISTEN_HOST:$http_port/" --data-binary "$1"; } +siblings() { + for core in ${cores//,/ }; do + printf '%s:%s ' "$core" "$(cat "/sys/devices/system/cpu/cpu$core/topology/thread_siblings_list")" + done +} +{ echo "binary_sha256 $actual" + echo "version $(query "SELECT version()")" + echo "git_hash $(query "SELECT value FROM system.build_options WHERE name = 'VERSION_GITHASH'")" + echo "base_image $BASE_IMAGE" + echo "cpuset $cores mems $numa" + echo "thread_siblings $(siblings)" + echo "cpu_model $(lscpu | sed -n 's/^Model name: *//p')" + echo "cgroup $group" + echo "kernel $(uname -r)"; } > "$data_dir/versions.txt" +cat "$data_dir/versions.txt" From ac7d4b32e05eb28e395971bdbbcf11aaf86264fe Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 17 Nov 2026 16:41:29 +0000 Subject: [PATCH 3/6] engines-external, cli: apply the independent review of the ClickHouse adapter - Ingest makes one pass over the chunk, with each source's insert statement built once at install; a chunk with a sample at or before 0 inserts nothing; label names are quoted as the translator quotes them; TSV escapes \r and reads \r, \b and \f back. - Install numbers its database from one counter for the whole process, so two engines in one process never share a database name. - The fake lane refuses the clickhouse flags. - extbench-up.sh: kills only its own agent, checks the agent answers, pins the agent with AGENT_CORES, resolves relative paths, gives the server's other protocols fixed ports on the private address (a port of 0 binds a random one), and never wipes a data dir it did not make (tested). - A failed install drops its database (tested); the README maps each capability row to its conformance case. --- aqpbm-cli/src/bench_external_cmd.rs | 7 +- aqpbm-engines-external/capabilities/README.md | 16 +++ aqpbm-engines-external/src/clickhouse.rs | 2 +- .../src/clickhouse_server.rs | 118 +++++++++++------- .../tests/clickhouse_conformance.rs | 7 +- .../tests/clickhouse_server.rs | 33 ++++- .../tests/extbench_script.rs | 29 +++++ scripts/extbench-up.sh | 39 +++++- 8 files changed, 194 insertions(+), 57 deletions(-) diff --git a/aqpbm-cli/src/bench_external_cmd.rs b/aqpbm-cli/src/bench_external_cmd.rs index a3994969..2a2f7729 100644 --- a/aqpbm-cli/src/bench_external_cmd.rs +++ b/aqpbm-cli/src/bench_external_cmd.rs @@ -108,7 +108,12 @@ pub fn run(args: BenchExternalArgs) -> Result<()> { fn run_lane_cmd(args: RunArgs) -> Result<()> { let engine: Box = match args.lane.as_str() { - "fake" => Box::new(FakeEngine::default()), + "fake" => { + if args.clickhouse_url.is_some() || args.cgroup_agent_url.is_some() { + bail!("--clickhouse-url and --cgroup-agent-url are for lane clickhouse only"); + } + Box::new(FakeEngine::default()) + } "clickhouse" => { let (Some(url), Some(cgroup_agent_url)) = (args.clickhouse_url.clone(), args.cgroup_agent_url.clone()) diff --git a/aqpbm-engines-external/capabilities/README.md b/aqpbm-engines-external/capabilities/README.md index 5142ad7d..0cc83f38 100644 --- a/aqpbm-engines-external/capabilities/README.md +++ b/aqpbm-engines-external/capabilities/README.md @@ -46,6 +46,22 @@ Prometheus `extrapolatedRate` is cited at v3.15.1. A stored read is right only at a window end, so a query that reads stored windows fails (`throwIf`) at any other evaluation time. Every insert uses `INSERT_SETTINGS`, so a NULL value is an error instead of 0. The sketch rows above give each sketch's class. The ClickHouse files list no sketch yet, because the translator does not build sketch states yet; each sketch joins `supported_summaries` in the PR that adds its translation rule. +**Conformance on a server.** `tests/clickhouse_conformance.rs` (`#[ignore]`, `AQPBM_CLICKHOUSE_URL` and `AQPBM_CGROUP_AGENT_URL`) runs one case per row through the engine adapter (`clickhouse_server`) against a real server, with hand-computed answers: + +| Row | Case | +|---|---| +| stored Sum / Count / Min / Max | `stored_sum_count_min_max` | +| stored Rate / Increase (b) | `stored_rate_is_promql_minus_the_first_value_after_each_reset` (three installs fed in turn, max_threads 1 and 16) | +| query-time Rate / Increase; raw samples retained | `query_time_rate_is_promql` | +| query-time grouped sums; `Aggregate` | `grouped_sums_quantiles_and_aggregates` | +| instant selector | `instant_selector` | +| top_k | `top_k_ties_and_nan` | +| regrouping merges, `Project`, `BinaryOp`, checked division | `average_as_sum_over_count_and_checked_division` | +| IRate; query-time results kept across evaluations (c) | `class_c_rows_are_refused` | +| raw samples retained (stored-only metric); retention | `storage_and_retention` | +| window boundary (left-open) | `the_harness_conformance_kit_passes` | +| sketches | not yet: no sketch rule | + ## VictoriaMetrics The VictoriaMetrics translator comes in a later PR. Until then, these files are the planned capabilities and are checked only by the planner run in the PR's manual check. diff --git a/aqpbm-engines-external/src/clickhouse.rs b/aqpbm-engines-external/src/clickhouse.rs index 9a864e4c..b54f6782 100644 --- a/aqpbm-engines-external/src/clickhouse.rs +++ b/aqpbm-engines-external/src/clickhouse.rs @@ -1447,7 +1447,7 @@ fn column_list(labels: &[String]) -> String { .join(", ") } -fn quoted(identifier: &str) -> String { +pub(crate) fn quoted(identifier: &str) -> String { format!("`{}`", identifier.replace('\\', "\\\\").replace('`', "\\`")) } diff --git a/aqpbm-engines-external/src/clickhouse_server.rs b/aqpbm-engines-external/src/clickhouse_server.rs index 23139755..a10abde7 100644 --- a/aqpbm-engines-external/src/clickhouse_server.rs +++ b/aqpbm-engines-external/src/clickhouse_server.rs @@ -16,7 +16,8 @@ //! CPU: the server runs in its own cgroup, read through the cgroup agent. //! The server's HTTP summary header has no CPU counter, so //! [`StepReport::engine_reported_cpu_ns`] is `None`; `scanned_bytes` is the -//! summed `read_bytes` of the step's statements. +//! summed `read_bytes` of the step's statements (for an insert, the bytes +//! of input it parsed). use std::collections::BTreeMap; use std::sync::atomic::{AtomicU64, Ordering}; @@ -28,7 +29,7 @@ use aqpbm_core::harness::engine::{ InstalledArm, PlanSummary, SampleChunk, SeriesLabels, StepReport, WorkloadSpec, }; -use crate::clickhouse::{translate, SourceTable, INSERT_SETTINGS, SERIES_IDENTITY}; +use crate::clickhouse::{quoted, translate, SourceTable, INSERT_SETTINGS, SERIES_IDENTITY}; use crate::plan_file::{PlanFile, PlanOperator}; /// How to reach the server and how to run its queries. @@ -49,11 +50,15 @@ pub struct ClickHouseEngine { server: ClickHouseServer, http: Http, description: EngineDescription, - /// Each install's database is `_`. + /// Each install's database is `_`, with `n` from + /// [`INSTALL_COUNT`]. database_prefix: String, - installs: AtomicU64, } +/// Installs made by this process, over every engine: numbers each install's +/// database. +static INSTALL_COUNT: AtomicU64 = AtomicU64::new(0); + impl ClickHouseEngine { /// Connects to the server and reads its version and git hash. A /// database of an earlier run (`bench_*`) is refused: dropping it, or its @@ -92,7 +97,6 @@ impl ClickHouseEngine { server, http, description, - installs: AtomicU64::new(0), }) } } @@ -141,7 +145,7 @@ impl EngineUnderTest for ClickHouseEngine { let database = format!( "{}_{}", self.database_prefix, - self.installs.fetch_add(1, Ordering::Relaxed) + INSTALL_COUNT.fetch_add(1, Ordering::Relaxed) ); self.http .run(&format!("CREATE DATABASE {database}"), &[], None)?; @@ -150,7 +154,14 @@ impl EngineUnderTest for ClickHouseEngine { http: self.http.clone(), database, max_threads: self.server.max_threads, - sources: deployment.sources.clone(), + sources: deployment + .sources + .iter() + .map(|source| SourceInsert { + statement: insert_statement(source), + source: source.clone(), + }) + .collect(), queries: query_ids .into_iter() .zip(deployment.queries.clone()) @@ -275,19 +286,26 @@ struct ClickHouseArm { http: Http, database: String, max_threads: u32, - sources: Vec, + /// Each source table and the statement that inserts into it. + sources: Vec, /// Per plan root: the workload query id and its SQL. queries: Vec<(String, String)>, stored_tables: Vec, summary: PlanSummary, } +/// A source table and its `INSERT … FORMAT TabSeparated` statement. +struct SourceInsert { + source: SourceTable, + statement: String, +} + impl ClickHouseArm { /// Runs `sql` in the arm's database with its `max_threads`. fn run( &self, sql: &str, - extra: &[(&str, &str)], + url_params: &[(&str, &str)], insert_rows: Option<&str>, ) -> Result<(String, u64), EngineError> { let threads = self.max_threads.to_string(); @@ -295,7 +313,7 @@ impl ClickHouseArm { ("database", self.database.as_str()), ("max_threads", threads.as_str()), ]; - params.extend_from_slice(extra); + params.extend_from_slice(url_params); self.http.run(sql, ¶ms, insert_rows) } } @@ -317,37 +335,47 @@ impl InstalledArm for ClickHouseArm { } fn ingest(&mut self, chunk: &SampleChunk) -> Result { - let mut scanned_bytes = 0; - for source in &self.sources { - let mut rows = String::new(); - for sample in &chunk.samples { - if sample.labels.get("__name__") != Some(&source.metric) { - continue; - } - if sample.ts_ms <= 0 { - return Err(EngineError(format!( - "a sample at {} ms: timestamps must be after 0", - sample.ts_ms - ))); - } - rows.push_str(&sample.ts_ms.to_string()); - rows.push('\t'); - rows.push_str(&float_text(sample.value)); - for label in &source.label_columns { - rows.push('\t'); - let value = if label == SERIES_IDENTITY { - series_identity(&sample.labels) - } else { - sample.labels.get(label).cloned().unwrap_or_default() - }; - push_tsv_escaped(&mut rows, &value); + // A stored window's key is `(intDiv(ts - 1, w) + 1) * w`, and + // `intDiv` truncates toward zero, so a sample at or before 0 would + // land in the wrong window. Checked before any insert, so a refused + // chunk inserts nothing. + if let Some(sample) = chunk.samples.iter().find(|sample| sample.ts_ms <= 0) { + return Err(EngineError(format!( + "a sample at {} ms: timestamps must be after 0", + sample.ts_ms + ))); + } + // One pass over the chunk: each sample's rows go to its metric's + // source table; a metric the plan does not read is skipped. + let mut rows: Vec = vec![String::new(); self.sources.len()]; + for sample in &chunk.samples { + let Some(at) = sample.labels.get("__name__").and_then(|metric| { + self.sources + .iter() + .position(|insert| &insert.source.metric == metric) + }) else { + continue; + }; + let out = &mut rows[at]; + out.push_str(&sample.ts_ms.to_string()); + out.push('\t'); + out.push_str(&float_text(sample.value)); + for label in &self.sources[at].source.label_columns { + out.push('\t'); + if label == SERIES_IDENTITY { + push_tsv_escaped(out, &series_identity(&sample.labels)); + } else { + push_tsv_escaped(out, sample.labels.get(label).map_or("", String::as_str)); } - rows.push('\n'); } + out.push('\n'); + } + let mut scanned_bytes = 0; + for (insert, rows) in self.sources.iter().zip(&rows) { if rows.is_empty() { continue; } - let (_, read_bytes) = self.run(&insert_statement(source), &[], Some(&rows))?; + let (_, read_bytes) = self.run(&insert.statement, &[], Some(rows))?; scanned_bytes += read_bytes; } Ok(StepReport { @@ -401,7 +429,11 @@ impl InstalledArm for ClickHouseArm { .map_err(|_| EngineError(format!("unexpected parts row '{line}'")))?; if self.stored_tables.iter().any(|t| t == table) { footprint.view_bytes += bytes; - } else if self.sources.iter().any(|s| s.table == table) { + } else if self + .sources + .iter() + .any(|insert| insert.source.table == table) + { footprint.raw_retained_bytes += bytes; } } @@ -418,11 +450,7 @@ fn insert_statement(source: &SourceTable) -> String { let selected: Vec = (0..source.label_columns.len()) .map(|i| format!("c{i}")) .collect(); - let columns: Vec = source - .label_columns - .iter() - .map(|label| format!("`{}`", label.replace('`', "\\`"))) - .collect(); + let columns: Vec = source.label_columns.iter().map(|l| quoted(l)).collect(); format!( "INSERT INTO {table} (ts, value, {columns}) SETTINGS {INSERT_SETTINGS} \ SELECT fromUnixTimestamp64Milli(ts_ms), value, {selected} \ @@ -455,6 +483,7 @@ fn push_tsv_escaped(out: &mut String, value: &str) { '\\' => out.push_str("\\\\"), '\t' => out.push_str("\\t"), '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), c => out.push(c), } } @@ -471,6 +500,9 @@ fn tsv_unescaped(value: &str) -> String { match chars.next() { Some('t') => out.push('\t'), Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + Some('b') => out.push('\u{8}'), + Some('f') => out.push('\u{c}'), Some('0') => out.push('\0'), Some(other) => out.push(other), None => out.push('\\'), @@ -487,6 +519,7 @@ fn parse_answer(text: &str) -> Result, String> { return Ok(BTreeMap::new()); }; let names: Vec = header.split('\t').map(tsv_unescaped).collect(); + let identity = names.iter().position(|name| name == SERIES_IDENTITY); let mut series = BTreeMap::new(); for line in lines { let fields: Vec = line.split('\t').map(tsv_unescaped).collect(); @@ -494,7 +527,6 @@ fn parse_answer(text: &str) -> Result, String> { return Err(format!("row '{line}' does not match the header '{header}'")); } let (value, labels) = fields.split_last().expect("at least the value"); - let identity = names.iter().position(|name| name == SERIES_IDENTITY); let key: SeriesLabels = match identity { Some(at) => serde_json::from_str(&labels[at]) .map_err(|e| format!("series identity '{}': {e}", labels[at]))?, diff --git a/aqpbm-engines-external/tests/clickhouse_conformance.rs b/aqpbm-engines-external/tests/clickhouse_conformance.rs index 7da693a0..2d92fc51 100644 --- a/aqpbm-engines-external/tests/clickhouse_conformance.rs +++ b/aqpbm-engines-external/tests/clickhouse_conformance.rs @@ -26,9 +26,6 @@ use aqpbm_core::harness::engine::{ use common::{fixture_path, fixture_value, fixture_value_public, lcm, payload_body}; use serde_json::{json, Value}; -const IGNORE: &str = - "needs a ClickHouse 26.8.39.13 server (AQPBM_CLICKHOUSE_URL, AQPBM_CGROUP_AGENT_URL)"; - fn server(max_threads: u32) -> ClickHouseServer { ClickHouseServer { url: std::env::var("AQPBM_CLICKHOUSE_URL").expect("AQPBM_CLICKHOUSE_URL"), @@ -161,7 +158,6 @@ fn assert_rows(got: &BTreeMap, expected: &[(&str, f64)]) { #[test] #[ignore = "needs a ClickHouse server"] fn the_server_is_the_pinned_binary() { - let _ = IGNORE; let description: EngineDescription = engine(1).description(); assert_eq!( description.version, @@ -524,7 +520,7 @@ fn average_as_sum_over_count_and_checked_division() { 30_000, ), ); - let error = overflow.evaluate(120_000).err().expect("an error"); + let error = overflow.evaluate(120_000).expect_err("an error"); assert!(error.0.contains("checked division"), "{error}"); } @@ -580,6 +576,7 @@ fn storage_and_retention() { .query("database", "conformance_ttl") .send_string(sql) .map(|r| r.into_string().unwrap()) + .map_err(|e| e.to_string()) }; ureq::post(&format!("{url}/")) .send_string("CREATE DATABASE IF NOT EXISTS conformance_ttl") diff --git a/aqpbm-engines-external/tests/clickhouse_server.rs b/aqpbm-engines-external/tests/clickhouse_server.rs index c7b56d0b..72e39701 100644 --- a/aqpbm-engines-external/tests/clickhouse_server.rs +++ b/aqpbm-engines-external/tests/clickhouse_server.rs @@ -564,7 +564,7 @@ fn evaluate_parses_series_and_values() { .contains("FORMAT TabSeparatedWithNames") .then(|| reply.clone()) })]); - let error = engine_for(&stub).evaluate(600_000).err().expect("an error"); + let error = engine_for(&stub).evaluate(600_000).expect_err("an error"); assert!(error.0.contains("not a window end"), "{error}"); } } @@ -655,3 +655,34 @@ fn leftover_databases_are_refused() { .expect("refused"); assert!(error.0.contains("bench_1_0, bench_1_1"), "{error}"); } + +#[test] +fn a_failed_install_drops_its_database() { + // The database exists once CREATE DATABASE ran; a table that fails + // drops it again. + let stub = start_stub(vec![Box::new(|request: &Request| { + request.statement().starts_with("CREATE TABLE").then(|| { + ( + 500, + vec![], + "Code: 57. DB::Exception: table exists".to_string(), + ) + }) + })]); + let engine = engine(&stub); + let error = engine + .install( + &workload_of("sot_1h_stored_sum", 60_000, 0), + &arm("sot_1h_stored_sum"), + ) + .err() + .expect("refused"); + assert!(error.0.contains("table exists"), "{error}"); + let statements = statements(&stub); + let database = statements + .iter() + .find_map(|s| s.strip_prefix("CREATE DATABASE ")) + .expect("a database was created") + .to_string(); + assert!(statements.contains(&format!("DROP DATABASE IF EXISTS {database} SYNC"))); +} diff --git a/aqpbm-engines-external/tests/extbench_script.rs b/aqpbm-engines-external/tests/extbench_script.rs index 3335289e..510c48bc 100644 --- a/aqpbm-engines-external/tests/extbench_script.rs +++ b/aqpbm-engines-external/tests/extbench_script.rs @@ -42,3 +42,32 @@ fn only_private_listen_addresses_are_accepted() { assert_eq!(out.status.success(), accepted, "{address}"); } } + +#[test] +fn a_data_dir_the_script_did_not_make_is_never_wiped() { + let dir = std::env::temp_dir().join(format!("extbench-guard-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("keep"), "data").unwrap(); + let binary = "/bin/true"; + let sha256 = Command::new("sha256sum").arg(binary).output().unwrap(); + let sha256 = String::from_utf8(sha256.stdout).unwrap(); + let out = Command::new("bash") + .arg(script()) + .args(["clickhouse", "up"]) + .arg(&dir) + .args(["2,22", "0", "18123", "19101"]) + .env("LISTEN_HOST", "127.0.0.1") + .env("CLICKHOUSE_BINARY", binary) + .env( + "CLICKHOUSE_SHA256", + sha256.split_whitespace().next().unwrap(), + ) + .env("BASE_IMAGE", "unused") + .env("APPROXBENCH", "unused") + .output() + .unwrap(); + assert!(!out.status.success()); + assert!(String::from_utf8_lossy(&out.stderr).contains("was not made by this script")); + assert!(dir.join("keep").exists()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/scripts/extbench-up.sh b/scripts/extbench-up.sh index e17c0f11..c979482f 100755 --- a/scripts/extbench-up.sh +++ b/scripts/extbench-up.sh @@ -18,13 +18,20 @@ # CLICKHOUSE_SHA256 its expected sha256 (required); a mismatch refuses # BASE_IMAGE the image the binary runs in, pinned by digest (required) # APPROXBENCH path of the approxbench binary, for `cgroup-agent` (required) +# AGENT_CORES cores the agent may run on (default: every core); keep +# them off the server's cores +# +# The agent runs outside the container, so its reads are never charged to +# the server's group. # # Every start begins from an empty data dir. The script only deletes a data # dir it created (it holds the marker file .extbench-data-dir). The # container runs as the invoking user, with no CPU quota (--cpus unset) and # no memory limit. ClickHouse runs on its embedded configuration: without a # main config file it reads no config.d, so the few settings changed here -# are `--key=value` overrides. +# are `--key=value` overrides. The embedded configuration also defines the +# native, MySQL, PostgreSQL and interserver ports (a port of 0 binds a random +# one), so they get the four ports after the HTTP port, on LISTEN_HOST too. set -euo pipefail marker=.extbench-data-dir @@ -48,12 +55,22 @@ if [ "${1:-}" = check-listen-host ]; then exit 0 fi -engine=${1:?engine}; action=${2:?up|down}; data_dir=${3:?data dir} +engine=${1:?engine}; action=${2:?up|down}; data_dir=$(realpath -m "${3:?data dir}") + +# Kill the agent recorded in $data_dir/agent.pid, if that pid is still it. +stop_agent() { + local pid + [ -f "$data_dir/agent.pid" ] || return 0 + pid=$(cat "$data_dir/agent.pid") + if [ -r "/proc/$pid/cmdline" ] && tr '\0' ' ' < "/proc/$pid/cmdline" | grep -q cgroup-agent; then + kill "$pid" 2>/dev/null || true + fi +} [ "$engine" = clickhouse ] || { echo "unknown engine $engine" >&2; exit 2; } if [ "$action" = down ]; then name=${4:-extbench-clickhouse} - [ -f "$data_dir/agent.pid" ] && kill "$(cat "$data_dir/agent.pid")" 2>/dev/null || true + stop_agent docker rm -f "$name" >/dev/null 2>&1 || true exit 0 fi @@ -63,12 +80,13 @@ name=${8:-extbench-clickhouse} : "${LISTEN_HOST:?set LISTEN_HOST to a private address}" "${CLICKHOUSE_BINARY:?}" \ "${CLICKHOUSE_SHA256:?}" "${BASE_IMAGE:?}" "${APPROXBENCH:?}" private_address "$LISTEN_HOST" || { echo "refused: LISTEN_HOST $LISTEN_HOST is not a private address" >&2; exit 1; } +CLICKHOUSE_BINARY=$(realpath "$CLICKHOUSE_BINARY") actual=$(sha256sum "$CLICKHOUSE_BINARY" | cut -d' ' -f1) [ "$actual" = "$CLICKHOUSE_SHA256" ] || { echo "binary sha256 $actual != $CLICKHOUSE_SHA256" >&2; exit 1; } if [ -e "$data_dir" ]; then [ -f "$data_dir/$marker" ] || { echo "refused: $data_dir exists and was not made by this script" >&2; exit 1; } - [ -f "$data_dir/agent.pid" ] && kill "$(cat "$data_dir/agent.pid")" 2>/dev/null || true + stop_agent docker rm -f "$name" >/dev/null 2>&1 || true rm -rf "$data_dir" fi @@ -80,7 +98,8 @@ docker run -d --name "$name" --network host \ -v "$CLICKHOUSE_BINARY:/usr/bin/clickhouse:ro" -v "$data_dir/db:/var/lib/clickhouse" \ -w /var/lib/clickhouse \ "$BASE_IMAGE" /usr/bin/clickhouse server -- --path=/var/lib/clickhouse/ \ - --http_port="$http_port" --tcp_port=0 --mysql_port=0 --postgresql_port=0 --interserver_http_port=0 \ + --http_port="$http_port" --tcp_port=$((http_port + 1)) --mysql_port=$((http_port + 2)) \ + --postgresql_port=$((http_port + 3)) --interserver_http_port=$((http_port + 4)) \ --listen_host="$LISTEN_HOST" --logger.level=warning --display_name="$name" >/dev/null for _ in $(seq 1 60); do curl -sf "http://$LISTEN_HOST:$http_port/ping" >/dev/null && break; sleep 1 @@ -90,8 +109,15 @@ curl -sf "http://$LISTEN_HOST:$http_port/ping" >/dev/null || { echo "the server id=$(docker inspect -f '{{.Id}}' "$name") group="/sys/fs/cgroup/system.slice/docker-$id.scope" [ -d "$group" ] || { echo "no cgroup at $group (is docker on the systemd cgroup driver?)" >&2; exit 1; } -nohup "$APPROXBENCH" cgroup-agent --cgroup-dir "$group" --listen "$LISTEN_HOST:$agent_port" \ +pin=() +[ -n "${AGENT_CORES:-}" ] && pin=(taskset -c "$AGENT_CORES") +nohup "${pin[@]}" "$APPROXBENCH" cgroup-agent --cgroup-dir "$group" --listen "$LISTEN_HOST:$agent_port" \ > "$data_dir/agent.log" 2>&1 & echo $! > "$data_dir/agent.pid" +for _ in $(seq 1 20); do + curl -sf "http://$LISTEN_HOST:$agent_port/cpu.stat" >/dev/null && break; sleep 0.5 +done +curl -sf "http://$LISTEN_HOST:$agent_port/cpu.stat" >/dev/null \ + || { echo "the cgroup agent did not start:" >&2; cat "$data_dir/agent.log" >&2; exit 1; } query() { curl -sf "http://$LISTEN_HOST:$http_port/" --data-binary "$1"; } siblings() { @@ -107,5 +133,6 @@ siblings() { echo "thread_siblings $(siblings)" echo "cpu_model $(lscpu | sed -n 's/^Model name: *//p')" echo "cgroup $group" + echo "agent_cores ${AGENT_CORES:-all}" echo "kernel $(uname -r)"; } > "$data_dir/versions.txt" cat "$data_dir/versions.txt" From 831eca4755ecc1b312e7d9a29e8e9b4ec700101f Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 17 Nov 2026 17:05:53 +0000 Subject: [PATCH 4/6] engines-external: failing test: answers drop __name__ unless the root is a bare selector PromQL's output labels: a function or aggregation drops the metric name, and only a bare selector keeps it. The report joins lanes by exact label maps, so the ClickHouse lane must key answers that way. --- .../tests/clickhouse_server.rs | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/aqpbm-engines-external/tests/clickhouse_server.rs b/aqpbm-engines-external/tests/clickhouse_server.rs index 72e39701..7eea1909 100644 --- a/aqpbm-engines-external/tests/clickhouse_server.rs +++ b/aqpbm-engines-external/tests/clickhouse_server.rs @@ -490,8 +490,11 @@ fn evaluate_parses_series_and_values() { ) .unwrap(); let (answers, report) = installed.evaluate(600_000).unwrap(); - let series: SeriesLabels = serde_json::from_str(identity).unwrap(); - assert!(answers.get("q0", &series).unwrap().is_nan()); + // The root applies a function (max_over_time), so the answer drops + // `__name__`, as PromQL's output does. + let mut series: SeriesLabels = serde_json::from_str(identity).unwrap(); + series.remove("__name__"); + assert!(answers.get("q0", &series).unwrap().is_nan(), "{answers:?}"); assert_eq!(report.scanned_bytes, Some(48)); assert_eq!(report.engine_reported_cpu_ns, None); let query = stub @@ -686,3 +689,43 @@ fn a_failed_install_drops_its_database() { .to_string(); assert!(statements.contains(&format!("DROP DATABASE IF EXISTS {database} SYNC"))); } + +#[test] +fn only_a_bare_selector_keeps_the_metric_name() { + // rangeheavy node 1 is the instant selector `data` (5 ms lookback): a + // root that is only that selector keeps `__name__`, as PromQL does. + let identity = r#"{"__name__":"data","job":"a"}"#; + let answer = format!("job\t$promql_series_identity\tvalue\na\t{identity}\t3\n"); + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| (200, vec![], answer.clone())) + })]); + let mut plan = common::fixture_value_public("rangeheavy", "clickhouse.equivalent-only"); + plan["dag"]["roots"] = json!([1]); + plan["query_texts"] = json!(["data"]); + let dir = std::env::temp_dir().join(format!("ch-server-selector-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("selector.plan.json"); + std::fs::write(&path, plan.to_string()).unwrap(); + let workload = WorkloadSpec { + name: "selector".into(), + queries: vec![WorkloadQuery { + id: "q0".into(), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: "data".into(), + }], + window_ms: 60_000, + evaluation_interval_ms: 60_000, + }], + data: vec![data_record(0)], + }; + let mut selector_arm = arm("sot_1h_stored_sum"); + selector_arm.engine_options = json!({ "plan_file": path }); + let mut installed = engine(&stub).install(&workload, &selector_arm).unwrap(); + let (answers, _) = installed.evaluate(60_000).unwrap(); + let series: SeriesLabels = serde_json::from_str(identity).unwrap(); + assert_eq!(answers.get("q0", &series), Some(3.0), "{answers:?}"); +} From 965f7109c292039bad32ac25876a5b4892584994 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 17 Nov 2026 17:08:14 +0000 Subject: [PATCH 5/6] engines-external: answers keep __name__ only for a bare selector The ClickHouse lane keys answers by PromQL's output labels: a root that applies a function or aggregation drops the metric name, and a root that is only an instant selector keeps it. --- .../src/clickhouse_server.rs | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/aqpbm-engines-external/src/clickhouse_server.rs b/aqpbm-engines-external/src/clickhouse_server.rs index a10abde7..53479eb8 100644 --- a/aqpbm-engines-external/src/clickhouse_server.rs +++ b/aqpbm-engines-external/src/clickhouse_server.rs @@ -8,10 +8,11 @@ //! queries with `param_t=`. //! //! Series: a sample's series identity is its full label map as compact -//! JSON with sorted keys, e.g. `{"__name__":"latency","job":"a"}`. A -//! per-series answer is keyed by that map (`__name__` kept, as the harness -//! keys answers). An aggregated answer is keyed by its label columns, with -//! empty values left out, as PromQL leaves out an empty label. +//! JSON with sorted keys, e.g. `{"__name__":"latency","job":"a"}`. Answers +//! carry PromQL's output labels: a per-series answer is keyed by that map, +//! and an aggregated one by its label columns with empty values left out, +//! as PromQL leaves out an empty label; `__name__` is kept only by a root +//! that is a bare selector, since a function or aggregation drops it. //! //! CPU: the server runs in its own cgroup, read through the cgroup agent. //! The server's HTTP summary header has no CPU counter, so @@ -30,7 +31,7 @@ use aqpbm_core::harness::engine::{ }; use crate::clickhouse::{quoted, translate, SourceTable, INSERT_SETTINGS, SERIES_IDENTITY}; -use crate::plan_file::{PlanFile, PlanOperator}; +use crate::plan_file::{PlanFile, PlanOperator, RangeKind}; /// How to reach the server and how to run its queries. #[derive(Debug, Clone, PartialEq, Eq)] @@ -164,7 +165,19 @@ impl EngineUnderTest for ClickHouseEngine { .collect(), queries: query_ids .into_iter() - .zip(deployment.queries.clone()) + .zip(&deployment.queries) + .zip(&plan.roots) + .map(|((id, sql), &root)| RootQuery { + id, + sql: sql.clone(), + keeps_metric_name: matches!( + plan.nodes[root].operator, + PlanOperator::TimeRange { + kind: RangeKind::Instant, + .. + } + ), + }) .collect(), stored_tables: deployment.stored_tables.clone(), summary: plan_summary(&plan, &deployment), @@ -288,12 +301,23 @@ struct ClickHouseArm { max_threads: u32, /// Each source table and the statement that inserts into it. sources: Vec, - /// Per plan root: the workload query id and its SQL. - queries: Vec<(String, String)>, + /// Per plan root: the workload query id, its SQL, and whether its + /// answers keep `__name__`. + queries: Vec, stored_tables: Vec, summary: PlanSummary, } +/// One query root as the arm runs it. +struct RootQuery { + id: String, + sql: String, + /// PromQL's output labels: only a bare selector keeps `__name__`; a + /// function or aggregation drops it, e.g. `quantile_over_time(0.99, + /// latency[1m])` answers `{instance="i1", job="a"}`. + keeps_metric_name: bool, +} + /// A source table and its `INSERT … FORMAT TabSeparated` statement. struct SourceInsert { source: SourceTable, @@ -388,7 +412,12 @@ impl InstalledArm for ClickHouseArm { let t = at_ms.to_string(); let mut answers = BTreeMap::new(); let mut scanned_bytes = 0; - for (id, sql) in &self.queries { + for RootQuery { + id, + sql, + keeps_metric_name, + } in &self.queries + { let (text, read_bytes) = self.run( &format!("{sql} FORMAT TabSeparatedWithNames"), &[("param_t", &t)], @@ -397,7 +426,8 @@ impl InstalledArm for ClickHouseArm { scanned_bytes += read_bytes; answers.insert( id.clone(), - parse_answer(&text).map_err(|e| EngineError(format!("query {id}: {e}")))?, + parse_answer(&text, *keeps_metric_name) + .map_err(|e| EngineError(format!("query {id}: {e}")))?, ); } Ok(( @@ -513,7 +543,10 @@ fn tsv_unescaped(value: &str) -> String { /// The rows of one query answer (`TabSeparatedWithNames`, the value /// column last) as series and values. -fn parse_answer(text: &str) -> Result, String> { +fn parse_answer( + text: &str, + keeps_metric_name: bool, +) -> Result, String> { let mut lines = text.lines(); let Some(header) = lines.next() else { return Ok(BTreeMap::new()); @@ -527,7 +560,7 @@ fn parse_answer(text: &str) -> Result, String> { return Err(format!("row '{line}' does not match the header '{header}'")); } let (value, labels) = fields.split_last().expect("at least the value"); - let key: SeriesLabels = match identity { + let mut key: SeriesLabels = match identity { Some(at) => serde_json::from_str(&labels[at]) .map_err(|e| format!("series identity '{}': {e}", labels[at]))?, None => names @@ -537,6 +570,9 @@ fn parse_answer(text: &str) -> Result, String> { .map(|(name, value)| (name.clone(), value.clone())) .collect(), }; + if !keeps_metric_name { + key.remove("__name__"); + } let value = parse_float(value).ok_or_else(|| format!("value '{value}' is not a number"))?; if series.insert(key.clone(), value).is_some() { return Err(format!("two rows for the series {key:?}")); From c9e50420c6b6940063c17783d8a5889fea0bbecb Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 17 Nov 2026 20:00:38 +0000 Subject: [PATCH 6/6] engines-external, cli: rustfmt after the move into aqpbm-core --- aqpbm-cli/src/bench_external_cmd.rs | 2 +- aqpbm-engines-external/tests/clickhouse_conformance.rs | 10 ++++++---- aqpbm-engines-external/tests/clickhouse_server.rs | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/aqpbm-cli/src/bench_external_cmd.rs b/aqpbm-cli/src/bench_external_cmd.rs index 2a2f7729..9b11778b 100644 --- a/aqpbm-cli/src/bench_external_cmd.rs +++ b/aqpbm-cli/src/bench_external_cmd.rs @@ -9,7 +9,6 @@ use std::net::TcpListener; use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; -use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; use aqpbm_core::harness::bench_external::{report, run_lane, write_report, LaneSetup}; use aqpbm_core::harness::cgroup::{agent_in_served_group, serve_agent}; use aqpbm_core::harness::engine::EngineUnderTest; @@ -18,6 +17,7 @@ use aqpbm_core::harness::protocol::{ load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, }; use aqpbm_core::harness::rundir::RunDir; +use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; use clap::{Parser, Subcommand}; /// The lanes this binary can run. diff --git a/aqpbm-engines-external/tests/clickhouse_conformance.rs b/aqpbm-engines-external/tests/clickhouse_conformance.rs index 2d92fc51..d4a3837b 100644 --- a/aqpbm-engines-external/tests/clickhouse_conformance.rs +++ b/aqpbm-engines-external/tests/clickhouse_conformance.rs @@ -15,14 +15,14 @@ use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use aqpbm_engines_external::clickhouse::translate; -use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; -use aqpbm_engines_external::plan_file::PlanFile; use aqpbm_core::harness::data::{workload_chunks, DataRecord, Sample}; use aqpbm_core::harness::engine::{ Answers, ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, QueryLanguage, QueryText, SampleChunk, SeriesLabels, WorkloadQuery, WorkloadSpec, }; +use aqpbm_engines_external::clickhouse::translate; +use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; +use aqpbm_engines_external::plan_file::PlanFile; use common::{fixture_path, fixture_value, fixture_value_public, lcm, payload_body}; use serde_json::{json, Value}; @@ -626,7 +626,9 @@ fn the_harness_conformance_kit_passes() { let report = aqpbm_core::harness::conformance::check(&mut engine, &clocks); println!("harness kit: {report:?}"); let passed = report.expect("the kit passes").passed; - assert!(passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::DeterministicAnswers)); + assert!( + passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::DeterministicAnswers) + ); assert!(passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::LeftOpenWindow)); } diff --git a/aqpbm-engines-external/tests/clickhouse_server.rs b/aqpbm-engines-external/tests/clickhouse_server.rs index 7eea1909..8cb93cdc 100644 --- a/aqpbm-engines-external/tests/clickhouse_server.rs +++ b/aqpbm-engines-external/tests/clickhouse_server.rs @@ -9,15 +9,15 @@ use std::net::TcpListener; use std::sync::{Arc, Mutex}; use std::time::Duration; -use aqpbm_engines_external::clickhouse_server::{ - series_identity, ClickHouseEngine, ClickHouseServer, -}; use aqpbm_core::harness::cgroup::CgroupSource; use aqpbm_core::harness::data::{DataRecord, Sample}; use aqpbm_core::harness::engine::{ ArmSpec, CpuAccounting, EngineUnderTest, QueryLanguage, QueryText, SampleChunk, SeriesLabels, WorkloadQuery, WorkloadSpec, }; +use aqpbm_engines_external::clickhouse_server::{ + series_identity, ClickHouseEngine, ClickHouseServer, +}; use common::{fixture_path, fixture_value, payload_body}; use serde_json::json;