diff --git a/Cargo.toml b/Cargo.toml index 32b0b6a6..b666e4b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", rev = # Internal workspace crates. aqpbm-datagen = { path = "aqpbm-datagen" } aqpbm-core = { path = "aqpbm-core", default-features = false } +aqpbm-engines-external = { path = "aqpbm-engines-external" } sketch-bench = { path = "sketch-bench" } aqpbm-usecase = { path = "aqpbm-usecase" } # sketch-runtime = { path = "sketch-runtime" } # parked; diff --git a/aqpbm-cli/Cargo.toml b/aqpbm-cli/Cargo.toml index 9343ff32..e38d3eb5 100644 --- a/aqpbm-cli/Cargo.toml +++ b/aqpbm-cli/Cargo.toml @@ -28,6 +28,7 @@ heap-track = ["sketch-bench/heap-track", "aqpbm-core/heap-track"] [dependencies] aqpbm-datagen = { workspace = true, features = ["import"] } aqpbm-core.workspace = true +aqpbm-engines-external.workspace = true sketch-bench.workspace = true clap.workspace = true serde.workspace = true diff --git a/aqpbm-cli/src/bench_external_cmd.rs b/aqpbm-cli/src/bench_external_cmd.rs index 3a00905d..9b11778b 100644 --- a/aqpbm-cli/src/bench_external_cmd.rs +++ b/aqpbm-cli/src/bench_external_cmd.rs @@ -17,10 +17,11 @@ use aqpbm_core::harness::protocol::{ load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, }; use aqpbm_core::harness::rundir::RunDir; +use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; use clap::{Parser, Subcommand}; /// The lanes this binary can run. -pub const LANES: [&str; 1] = ["fake"]; +pub const LANES: [&str; 2] = ["fake", "clickhouse"]; #[derive(Parser, Debug)] pub struct BenchExternalArgs { @@ -41,9 +42,23 @@ pub struct RunArgs { /// The run directory: workloads/, plans//, results/. #[arg(long)] pub run_dir: PathBuf, - /// The lane: `fake` (the harness's own fake engine). + /// The lane: `fake` (the harness's own fake engine) or `clickhouse`. #[arg(long)] pub lane: String, + /// The ClickHouse server's HTTP interface, e.g. `http://10.0.0.1:8123` + /// (lane `clickhouse`). + #[arg(long)] + pub clickhouse_url: Option, + /// The cgroup agent beside the server, e.g. `http://10.0.0.1:9101` + /// (lane `clickhouse`). + #[arg(long)] + pub cgroup_agent_url: Option, + /// `max_threads` of every ClickHouse statement; 1 for the main figures. + #[arg(long, default_value_t = 1)] + pub clickhouse_max_threads: u32, + /// How long one ClickHouse statement may run, in seconds. + #[arg(long, default_value_t = 600)] + pub clickhouse_statement_timeout_s: u64, /// Only these variants (repeatable); every variant under plans/ by default. #[arg(long = "variant")] pub variants: Vec, @@ -93,7 +108,27 @@ pub fn run(args: BenchExternalArgs) -> Result<()> { fn run_lane_cmd(args: RunArgs) -> Result<()> { let engine: Box = match args.lane.as_str() { - "fake" => Box::new(FakeEngine::default()), + "fake" => { + if args.clickhouse_url.is_some() || args.cgroup_agent_url.is_some() { + bail!("--clickhouse-url and --cgroup-agent-url are for lane clickhouse only"); + } + Box::new(FakeEngine::default()) + } + "clickhouse" => { + let (Some(url), Some(cgroup_agent_url)) = + (args.clickhouse_url.clone(), args.cgroup_agent_url.clone()) + else { + bail!("lane clickhouse needs --clickhouse-url and --cgroup-agent-url"); + }; + Box::new(ClickHouseEngine::new(ClickHouseServer { + url, + cgroup_agent_url, + max_threads: args.clickhouse_max_threads, + statement_timeout: std::time::Duration::from_secs( + args.clickhouse_statement_timeout_s, + ), + })?) + } other => bail!("unknown lane '{other}'; known lanes: {}", LANES.join(", ")), }; let run_dir = RunDir::open(&args.run_dir)?; diff --git a/aqpbm-cli/tests/bench_external_cli.rs b/aqpbm-cli/tests/bench_external_cli.rs index d74c1c9a..6bfc18be 100644 --- a/aqpbm-cli/tests/bench_external_cli.rs +++ b/aqpbm-cli/tests/bench_external_cli.rs @@ -152,7 +152,7 @@ fn an_unknown_lane_names_the_known_lanes() { "--run-dir", run.to_str().unwrap(), "--lane", - "clickhouse", + "victoriametrics", ]); assert!(!out.status.success()); let stderr = String::from_utf8_lossy(&out.stderr); @@ -162,3 +162,26 @@ fn an_unknown_lane_names_the_known_lanes() { ); let _ = std::fs::remove_dir_all(run.parent().unwrap()); } + +/// X3: the clickhouse lane needs the server and its cgroup agent. +#[test] +fn the_clickhouse_lane_needs_its_server() { + let run = fixture("x3"); + let out = approxbench(&[ + "bench-external", + "run", + "--run-dir", + run.to_str().unwrap(), + "--lane", + "clickhouse", + "--clickhouse-url", + "http://10.0.0.2:8123", + ]); + assert!(!out.status.success()); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("--clickhouse-url") && stderr.contains("--cgroup-agent-url"), + "{stderr}" + ); + let _ = std::fs::remove_dir_all(run.parent().unwrap()); +} diff --git a/aqpbm-engines-external/Cargo.toml b/aqpbm-engines-external/Cargo.toml index 84667e0a..ef4e94a9 100644 --- a/aqpbm-engines-external/Cargo.toml +++ b/aqpbm-engines-external/Cargo.toml @@ -7,5 +7,7 @@ repository.workspace = true description = "Runs ASAPPlanner plan files on external engines: reads an InstallablePlan file (wire version 15) and translates it into ClickHouse materialized views and SQL. The translator is the capability check: a node it has no rule for is refused. Depends on no ASAPPlanner crate." [dependencies] +aqpbm-core.workspace = true serde_json.workspace = true +ureq = { version = "2", default-features = false } thiserror.workspace = true diff --git a/aqpbm-engines-external/capabilities/README.md b/aqpbm-engines-external/capabilities/README.md index 5142ad7d..0cc83f38 100644 --- a/aqpbm-engines-external/capabilities/README.md +++ b/aqpbm-engines-external/capabilities/README.md @@ -46,6 +46,22 @@ Prometheus `extrapolatedRate` is cited at v3.15.1. A stored read is right only at a window end, so a query that reads stored windows fails (`throwIf`) at any other evaluation time. Every insert uses `INSERT_SETTINGS`, so a NULL value is an error instead of 0. The sketch rows above give each sketch's class. The ClickHouse files list no sketch yet, because the translator does not build sketch states yet; each sketch joins `supported_summaries` in the PR that adds its translation rule. +**Conformance on a server.** `tests/clickhouse_conformance.rs` (`#[ignore]`, `AQPBM_CLICKHOUSE_URL` and `AQPBM_CGROUP_AGENT_URL`) runs one case per row through the engine adapter (`clickhouse_server`) against a real server, with hand-computed answers: + +| Row | Case | +|---|---| +| stored Sum / Count / Min / Max | `stored_sum_count_min_max` | +| stored Rate / Increase (b) | `stored_rate_is_promql_minus_the_first_value_after_each_reset` (three installs fed in turn, max_threads 1 and 16) | +| query-time Rate / Increase; raw samples retained | `query_time_rate_is_promql` | +| query-time grouped sums; `Aggregate` | `grouped_sums_quantiles_and_aggregates` | +| instant selector | `instant_selector` | +| top_k | `top_k_ties_and_nan` | +| regrouping merges, `Project`, `BinaryOp`, checked division | `average_as_sum_over_count_and_checked_division` | +| IRate; query-time results kept across evaluations (c) | `class_c_rows_are_refused` | +| raw samples retained (stored-only metric); retention | `storage_and_retention` | +| window boundary (left-open) | `the_harness_conformance_kit_passes` | +| sketches | not yet: no sketch rule | + ## VictoriaMetrics The VictoriaMetrics translator comes in a later PR. Until then, these files are the planned capabilities and are checked only by the planner run in the PR's manual check. diff --git a/aqpbm-engines-external/src/clickhouse.rs b/aqpbm-engines-external/src/clickhouse.rs index 748ec43b..b54f6782 100644 --- a/aqpbm-engines-external/src/clickhouse.rs +++ b/aqpbm-engines-external/src/clickhouse.rs @@ -49,7 +49,7 @@ use crate::{NodeId, SharingLost, Substitution, TranslationError}; /// The plan's series identity column, a label column like any other. It has /// one value per series, so it is a plain `String`: `LowCardinality` pays /// off only below about ten thousand distinct values. -const SERIES_IDENTITY: &str = "$promql_series_identity"; +pub const SERIES_IDENTITY: &str = "$promql_series_identity"; /// The settings every `INSERT` into a source table uses: a NULL value is an /// error instead of being stored as 0 (PromQL samples are never NULL), in @@ -62,6 +62,11 @@ pub const INSERT_SETTINGS: &str = "input_format_null_as_default = 0, insert_null pub struct ClickHouseDeployment { /// `CREATE TABLE` statements: one per metric, then one per stored chain. pub tables: Vec, + /// The source table of each metric, in the order of `tables`: where its + /// samples are inserted. + pub sources: Vec, + /// The tables of the stored chains (`v`), in the order of `tables`. + pub stored_tables: Vec, /// `CREATE MATERIALIZED VIEW` statements, one per stored chain. pub materialized_views: Vec, /// One `SELECT` per query root, in root order, with the parameter `{t:Int64}`. @@ -77,6 +82,16 @@ pub struct ClickHouseDeployment { pub sharing_lost: Vec, } +/// A metric's source table: samples of `metric` (their `__name__`) are +/// inserted into `table`, with one column per label of `label_columns`, in +/// that order, the series identity last. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SourceTable { + pub metric: String, + pub table: String, + pub label_columns: Vec, +} + /// A table's retention: rows whose time is older than `retention_ms` before /// the newest evaluation are no longer read. #[derive(Debug, Clone, PartialEq, Eq)] @@ -136,12 +151,15 @@ pub fn translate(plan: &PlanFile) -> Result, Vec) = + source_tables(plan).into_iter().unzip(); + let mut stored_tables = Vec::new(); let mut table_retention = source_retention(plan); let mut materialized_views = Vec::new(); let mut substitutions = Vec::new(); for chain in translator.chains.values() { tables.push(chain.table()); + stored_tables.push(chain.table_name()); materialized_views.push(chain.view()); if let Some(retention_ms) = chain.retention_ms { table_retention.push(TableRetention { @@ -161,6 +179,8 @@ pub fn translate(plan: &PlanFile) -> Result String { /// primary key; a `minmax` index on `ts` skips the granules outside a time /// range. A metric only stored chains read keeps no rows (`ENGINE = Null`): /// its materialized views still see every insert. -fn source_tables(plan: &PlanFile) -> Vec { +fn source_tables(plan: &PlanFile) -> Vec<(SourceTable, String)> { struct SourceColumns<'a> { metric: &'a str, labels: Vec<&'a str>, @@ -421,10 +441,18 @@ fn source_tables(plan: &PlanFile) -> Vec { .map(|label| format!(", {} {}", quoted(label), label_type(label))) .collect(); let table = source_table(source.metric); + let described = SourceTable { + metric: source.metric.to_string(), + table: table.clone(), + label_columns: source.labels.iter().map(|l| l.to_string()).collect(), + }; if !source.keeps_rows { - return format!( - "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}) \ - ENGINE = Null" + return ( + described, + format!( + "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}) \ + ENGINE = Null" + ), ); } let sorting_key: Vec = source @@ -434,11 +462,14 @@ fn source_tables(plan: &PlanFile) -> Vec { .map(|label| quoted(label)) .chain(["ts".to_string()]) .collect(); - format!( - "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}, \ - INDEX ts_minmax ts TYPE minmax GRANULARITY 1) \ - ENGINE = MergeTree ORDER BY ({})", - sorting_key.join(", ") + ( + described, + format!( + "CREATE TABLE {table} (ts DateTime64(3, 'UTC'), value Float64{columns}, \ + INDEX ts_minmax ts TYPE minmax GRANULARITY 1) \ + ENGINE = MergeTree ORDER BY ({})", + sorting_key.join(", ") + ), ) }) .collect() @@ -1416,7 +1447,7 @@ fn column_list(labels: &[String]) -> String { .join(", ") } -fn quoted(identifier: &str) -> String { +pub(crate) fn quoted(identifier: &str) -> String { format!("`{}`", identifier.replace('\\', "\\\\").replace('`', "\\`")) } diff --git a/aqpbm-engines-external/src/clickhouse_server.rs b/aqpbm-engines-external/src/clickhouse_server.rs new file mode 100644 index 00000000..53479eb8 --- /dev/null +++ b/aqpbm-engines-external/src/clickhouse_server.rs @@ -0,0 +1,675 @@ +//! A ClickHouse server as an engine under test: each arm is a plan file +//! translated by [`crate::clickhouse::translate`] and installed in a fresh +//! database of the server over HTTP. +//! +//! Example: an arm whose `engine_options` are `{"plan_file": "q4c.plan.json"}` +//! installs `src_latency` in database `bench_4711_0`. Each refresh chunk is +//! one `INSERT` per source table, and each evaluation at `t` runs the plan's +//! queries with `param_t=`. +//! +//! Series: a sample's series identity is its full label map as compact +//! JSON with sorted keys, e.g. `{"__name__":"latency","job":"a"}`. Answers +//! carry PromQL's output labels: a per-series answer is keyed by that map, +//! and an aggregated one by its label columns with empty values left out, +//! as PromQL leaves out an empty label; `__name__` is kept only by a root +//! that is a bare selector, since a function or aggregation drops it. +//! +//! CPU: the server runs in its own cgroup, read through the cgroup agent. +//! The server's HTTP summary header has no CPU counter, so +//! [`StepReport::engine_reported_cpu_ns`] is `None`; `scanned_bytes` is the +//! summed `read_bytes` of the step's statements (for an insert, the bytes +//! of input it parsed). + +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use aqpbm_core::harness::cgroup::CgroupSource; +use aqpbm_core::harness::engine::{ + Answers, ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, Footprint, + InstalledArm, PlanSummary, SampleChunk, SeriesLabels, StepReport, WorkloadSpec, +}; + +use crate::clickhouse::{quoted, translate, SourceTable, INSERT_SETTINGS, SERIES_IDENTITY}; +use crate::plan_file::{PlanFile, PlanOperator, RangeKind}; + +/// How to reach the server and how to run its queries. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClickHouseServer { + /// The HTTP interface, e.g. `http://10.0.0.1:8123`. + pub url: String, + /// The cgroup agent beside the server, e.g. `http://10.0.0.1:9101`. + pub cgroup_agent_url: String, + /// `max_threads` of every statement (1 for the main figures). + pub max_threads: u32, + /// How long one statement may take before it fails, e.g. 600 s. + pub statement_timeout: Duration, +} + +/// A ClickHouse server; see the module docs. +pub struct ClickHouseEngine { + server: ClickHouseServer, + http: Http, + description: EngineDescription, + /// Each install's database is `_`, with `n` from + /// [`INSTALL_COUNT`]. + database_prefix: String, +} + +/// Installs made by this process, over every engine: numbers each install's +/// database. +static INSTALL_COUNT: AtomicU64 = AtomicU64::new(0); + +impl ClickHouseEngine { + /// Connects to the server and reads its version and git hash. A + /// database of an earlier run (`bench_*`) is refused: dropping it, or its + /// background merges, would be charged to this run's measurements. + pub fn new(server: ClickHouseServer) -> Result { + let http = Http::new(&server.url, server.statement_timeout); + let (leftovers, _) = http.run( + "SELECT name FROM system.databases WHERE startsWith(name, 'bench_') FORMAT TabSeparated", + &[], + None, + )?; + if !leftovers.trim().is_empty() { + return Err(EngineError(format!( + "the server holds databases of an earlier run: {}; start it from an empty data dir", + leftovers.trim().replace('\n', ", ") + ))); + } + let (text, _) = http.run( + "SELECT version(), (SELECT value FROM system.build_options \ + WHERE name = 'VERSION_GITHASH') FORMAT TabSeparated", + &[], + None, + )?; + let (version, git_hash) = text + .trim_end() + .split_once('\t') + .ok_or_else(|| EngineError(format!("unexpected version answer '{text}'")))?; + let description = EngineDescription { + id: "clickhouse".into(), + version: format!("{version} ({git_hash})"), + version_dirty: false, + build_profile: format!("max_threads={}", server.max_threads), + }; + Ok(ClickHouseEngine { + database_prefix: format!("bench_{}", std::process::id()), + server, + http, + description, + }) + } +} + +impl EngineUnderTest for ClickHouseEngine { + fn description(&self) -> EngineDescription { + self.description.clone() + } + + fn cpu_accounting(&self) -> CpuAccounting { + CpuAccounting::Cgroup(CgroupSource::Agent { + url: self.server.cgroup_agent_url.clone(), + }) + } + + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let path = arm + .engine_options + .get("plan_file") + .and_then(|p| p.as_str()) + .ok_or_else(|| EngineError("engine_options has no plan_file".into()))?; + let text = std::fs::read_to_string(path) + .map_err(|error| EngineError(format!("plan file {path}: {error}")))?; + let plan = PlanFile::from_json(&text) + .map_err(|error| EngineError(format!("plan file {path}: {error}")))?; + let query_ids = query_ids_of(&plan, workload)?; + let deployment = + translate(&plan).map_err(|error| EngineError(format!("plan file {path}: {error}")))?; + check_evaluation_times(workload, &deployment.query_time_steps_ms)?; + if let Some(source) = deployment.sources.iter().find(|source| { + source + .label_columns + .iter() + .any(|label| label == "ts" || label == "value") + }) { + return Err(EngineError(format!( + "metric {} has a label named ts or value, the names of its sample columns", + source.metric + ))); + } + + let database = format!( + "{}_{}", + self.database_prefix, + INSTALL_COUNT.fetch_add(1, Ordering::Relaxed) + ); + self.http + .run(&format!("CREATE DATABASE {database}"), &[], None)?; + // From here on, dropping the arm drops the database. + let installed = ClickHouseArm { + http: self.http.clone(), + database, + max_threads: self.server.max_threads, + sources: deployment + .sources + .iter() + .map(|source| SourceInsert { + statement: insert_statement(source), + source: source.clone(), + }) + .collect(), + queries: query_ids + .into_iter() + .zip(&deployment.queries) + .zip(&plan.roots) + .map(|((id, sql), &root)| RootQuery { + id, + sql: sql.clone(), + keeps_metric_name: matches!( + plan.nodes[root].operator, + PlanOperator::TimeRange { + kind: RangeKind::Instant, + .. + } + ), + }) + .collect(), + stored_tables: deployment.stored_tables.clone(), + summary: plan_summary(&plan, &deployment), + }; + // TTL statements are never applied: the harness replays data, and a + // TTL deletes by wall-clock time. + for statement in deployment + .tables + .iter() + .chain(&deployment.materialized_views) + { + installed.run(statement, &[], None)?; + } + Ok(Box::new(installed)) + } +} + +/// Every evaluation time is a multiple of the evaluation interval from an +/// aligned data start ([`aqpbm_core::harness::arm_run::ArmSchedule`]), so a query +/// that reads stored windows answers right only when its step divides the +/// interval and the data start. Refused at install, not at the first +/// evaluation. +fn check_evaluation_times(workload: &WorkloadSpec, steps_ms: &[u64]) -> Result<(), EngineError> { + let data_start_ms = workload + .data + .iter() + .map(|record| record.description.start_ms) + .min() + .unwrap_or(0); + for (query, &step_ms) in workload.queries.iter().zip(steps_ms) { + let step = step_ms as i64; + if (query.evaluation_interval_ms as i64) % step != 0 || data_start_ms % step != 0 { + return Err(EngineError(format!( + "query {} reads stored windows of {step_ms} ms, which do not divide its \ + evaluation interval ({} ms) and the data start ({data_start_ms} ms)", + query.id, query.evaluation_interval_ms + ))); + } + } + Ok(()) +} + +/// The workload query id of each plan root: root i answers query i, whose +/// texts (in any language) must include the plan's text of root i. +fn query_ids_of(plan: &PlanFile, workload: &WorkloadSpec) -> Result, EngineError> { + if plan.query_texts.len() != workload.queries.len() { + return Err(EngineError(format!( + "the plan has {} queries but workload {} has {}", + plan.query_texts.len(), + workload.name, + workload.queries.len() + ))); + } + plan.query_texts + .iter() + .zip(&workload.queries) + .map(|(plan_text, query)| { + if query.texts.iter().any(|text| &text.text == plan_text) { + Ok(query.id.clone()) + } else { + let texts: Vec<&str> = query.texts.iter().map(|t| t.text.as_str()).collect(); + Err(EngineError(format!( + "query {}: the plan answers '{plan_text}', but the workload's texts are {texts:?}", + query.id + ))) + } + }) + .collect() +} + +fn plan_summary( + plan: &PlanFile, + deployment: &crate::clickhouse::ClickHouseDeployment, +) -> PlanSummary { + PlanSummary { + text: deployment.script(), + node_kinds: plan + .nodes + .iter() + .map(|node| operator_kind(&node.operator)) + .collect(), + stored_outputs: deployment.stored_tables.clone(), + substitutions: deployment + .substitutions + .iter() + .map(|s| format!("{} at node {}: {}", s.rule, s.node, s.guarantee_difference)) + .collect(), + sharing_lost: deployment + .sharing_lost + .iter() + .map(|lost| { + let readers: Vec = lost.readers.iter().map(|r| r.to_string()).collect(); + format!("node {} for nodes {}", lost.node, readers.join(", ")) + }) + .collect(), + } +} + +/// The planner's operator name of `operator`, e.g. `SummaryAgg`. +fn operator_kind(operator: &PlanOperator) -> String { + match operator { + PlanOperator::Scan { .. } => "Scan", + PlanOperator::TimeRange { .. } => "TimeRange", + PlanOperator::SummaryTumblingWindowChain { .. } => "SummaryTumblingWindowChain", + PlanOperator::SummaryAgg { .. } => "SummaryAgg", + PlanOperator::SummaryMerge { .. } => "SummaryMerge", + PlanOperator::FinalizeExactAccumulator { .. } => "FinalizeExactAccumulator", + PlanOperator::SummaryEstimate { .. } => "SummaryEstimate", + PlanOperator::Aggregate { .. } => "Aggregate", + PlanOperator::Project { .. } => "Project", + PlanOperator::BinaryOp { .. } => "BinaryOp", + PlanOperator::Other { kind } => kind, + } + .to_string() +} + +/// One installed arm: its database and the queries it answers. +struct ClickHouseArm { + http: Http, + database: String, + max_threads: u32, + /// Each source table and the statement that inserts into it. + sources: Vec, + /// Per plan root: the workload query id, its SQL, and whether its + /// answers keep `__name__`. + queries: Vec, + stored_tables: Vec, + summary: PlanSummary, +} + +/// One query root as the arm runs it. +struct RootQuery { + id: String, + sql: String, + /// PromQL's output labels: only a bare selector keeps `__name__`; a + /// function or aggregation drops it, e.g. `quantile_over_time(0.99, + /// latency[1m])` answers `{instance="i1", job="a"}`. + keeps_metric_name: bool, +} + +/// A source table and its `INSERT … FORMAT TabSeparated` statement. +struct SourceInsert { + source: SourceTable, + statement: String, +} + +impl ClickHouseArm { + /// Runs `sql` in the arm's database with its `max_threads`. + fn run( + &self, + sql: &str, + url_params: &[(&str, &str)], + insert_rows: Option<&str>, + ) -> Result<(String, u64), EngineError> { + let threads = self.max_threads.to_string(); + let mut params = vec![ + ("database", self.database.as_str()), + ("max_threads", threads.as_str()), + ]; + params.extend_from_slice(url_params); + self.http.run(sql, ¶ms, insert_rows) + } +} + +impl Drop for ClickHouseArm { + fn drop(&mut self) { + // Best effort: a server that is gone keeps nothing anyway. + let _ = self.http.run( + &format!("DROP DATABASE IF EXISTS {} SYNC", self.database), + &[], + None, + ); + } +} + +impl InstalledArm for ClickHouseArm { + fn plan_summary(&self) -> PlanSummary { + self.summary.clone() + } + + fn ingest(&mut self, chunk: &SampleChunk) -> Result { + // A stored window's key is `(intDiv(ts - 1, w) + 1) * w`, and + // `intDiv` truncates toward zero, so a sample at or before 0 would + // land in the wrong window. Checked before any insert, so a refused + // chunk inserts nothing. + if let Some(sample) = chunk.samples.iter().find(|sample| sample.ts_ms <= 0) { + return Err(EngineError(format!( + "a sample at {} ms: timestamps must be after 0", + sample.ts_ms + ))); + } + // One pass over the chunk: each sample's rows go to its metric's + // source table; a metric the plan does not read is skipped. + let mut rows: Vec = vec![String::new(); self.sources.len()]; + for sample in &chunk.samples { + let Some(at) = sample.labels.get("__name__").and_then(|metric| { + self.sources + .iter() + .position(|insert| &insert.source.metric == metric) + }) else { + continue; + }; + let out = &mut rows[at]; + out.push_str(&sample.ts_ms.to_string()); + out.push('\t'); + out.push_str(&float_text(sample.value)); + for label in &self.sources[at].source.label_columns { + out.push('\t'); + if label == SERIES_IDENTITY { + push_tsv_escaped(out, &series_identity(&sample.labels)); + } else { + push_tsv_escaped(out, sample.labels.get(label).map_or("", String::as_str)); + } + } + out.push('\n'); + } + let mut scanned_bytes = 0; + for (insert, rows) in self.sources.iter().zip(&rows) { + if rows.is_empty() { + continue; + } + let (_, read_bytes) = self.run(&insert.statement, &[], Some(rows))?; + scanned_bytes += read_bytes; + } + Ok(StepReport { + scanned_bytes: Some(scanned_bytes), + engine_reported_cpu_ns: None, + }) + } + + fn evaluate(&mut self, at_ms: i64) -> Result<(Answers, StepReport), EngineError> { + let t = at_ms.to_string(); + let mut answers = BTreeMap::new(); + let mut scanned_bytes = 0; + for RootQuery { + id, + sql, + keeps_metric_name, + } in &self.queries + { + let (text, read_bytes) = self.run( + &format!("{sql} FORMAT TabSeparatedWithNames"), + &[("param_t", &t)], + None, + )?; + scanned_bytes += read_bytes; + answers.insert( + id.clone(), + parse_answer(&text, *keeps_metric_name) + .map_err(|e| EngineError(format!("query {id}: {e}")))?, + ); + } + Ok(( + Answers(answers), + StepReport { + scanned_bytes: Some(scanned_bytes), + engine_reported_cpu_ns: None, + }, + )) + } + + fn footprint(&self) -> Result { + let (text, _) = self.run( + &format!( + "SELECT table, sum(bytes_on_disk) FROM system.parts \ + WHERE database = '{}' AND active GROUP BY table FORMAT TabSeparated", + self.database + ), + &[], + None, + )?; + let mut footprint = Footprint::default(); + for line in text.lines() { + let (table, bytes) = line + .split_once('\t') + .ok_or_else(|| EngineError(format!("unexpected parts row '{line}'")))?; + let bytes: u64 = bytes + .parse() + .map_err(|_| EngineError(format!("unexpected parts row '{line}'")))?; + if self.stored_tables.iter().any(|t| t == table) { + footprint.view_bytes += bytes; + } else if self + .sources + .iter() + .any(|insert| insert.source.table == table) + { + footprint.raw_retained_bytes += bytes; + } + } + Ok(footprint) + } +} + +/// `INSERT … SELECT … FROM input(…) FORMAT TabSeparated` into `source`: rows +/// of `ts_ms`, `value`, then one column per label column. +fn insert_statement(source: &SourceTable) -> String { + let inputs: Vec = (0..source.label_columns.len()) + .map(|i| format!("c{i} String")) + .collect(); + let selected: Vec = (0..source.label_columns.len()) + .map(|i| format!("c{i}")) + .collect(); + let columns: Vec = source.label_columns.iter().map(|l| quoted(l)).collect(); + format!( + "INSERT INTO {table} (ts, value, {columns}) SETTINGS {INSERT_SETTINGS} \ + SELECT fromUnixTimestamp64Milli(ts_ms), value, {selected} \ + FROM input('ts_ms Int64, value Float64, {inputs}') FORMAT TabSeparated", + table = source.table, + columns = columns.join(", "), + selected = selected.join(", "), + inputs = inputs.join(", "), + ) +} + +/// A series' identity: its label map as compact JSON, keys sorted. +pub fn series_identity(labels: &SeriesLabels) -> String { + serde_json::to_string(labels).expect("a string map serializes") +} + +/// `value` as TabSeparated reads it: `nan`, `inf`, `-inf`, or the shortest +/// decimal that reads back exactly. +fn float_text(value: f64) -> String { + if value.is_nan() { + "nan".into() + } else { + value.to_string() + } +} + +fn push_tsv_escaped(out: &mut String, value: &str) { + for c in value.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + c => out.push(c), + } + } +} + +fn tsv_unescaped(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('t') => out.push('\t'), + Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + Some('b') => out.push('\u{8}'), + Some('f') => out.push('\u{c}'), + Some('0') => out.push('\0'), + Some(other) => out.push(other), + None => out.push('\\'), + } + } + out +} + +/// The rows of one query answer (`TabSeparatedWithNames`, the value +/// column last) as series and values. +fn parse_answer( + text: &str, + keeps_metric_name: bool, +) -> Result, String> { + let mut lines = text.lines(); + let Some(header) = lines.next() else { + return Ok(BTreeMap::new()); + }; + let names: Vec = header.split('\t').map(tsv_unescaped).collect(); + let identity = names.iter().position(|name| name == SERIES_IDENTITY); + let mut series = BTreeMap::new(); + for line in lines { + let fields: Vec = line.split('\t').map(tsv_unescaped).collect(); + if fields.len() != names.len() { + return Err(format!("row '{line}' does not match the header '{header}'")); + } + let (value, labels) = fields.split_last().expect("at least the value"); + let mut key: SeriesLabels = match identity { + Some(at) => serde_json::from_str(&labels[at]) + .map_err(|e| format!("series identity '{}': {e}", labels[at]))?, + None => names + .iter() + .zip(labels) + .filter(|(_, value)| !value.is_empty()) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + }; + if !keeps_metric_name { + key.remove("__name__"); + } + let value = parse_float(value).ok_or_else(|| format!("value '{value}' is not a number"))?; + if series.insert(key.clone(), value).is_some() { + return Err(format!("two rows for the series {key:?}")); + } + } + Ok(series) +} + +fn parse_float(text: &str) -> Option { + match text { + "nan" | "-nan" => Some(f64::NAN), + "inf" | "+inf" => Some(f64::INFINITY), + "-inf" => Some(f64::NEG_INFINITY), + _ => text.parse().ok(), + } +} + +/// The server's HTTP interface. A statement is the `query` parameter; an +/// insert's rows are the body. +#[derive(Clone)] +struct Http { + agent: ureq::Agent, + url: String, + /// `max_execution_time` of every statement, in whole seconds. + max_execution_seconds: String, +} + +/// How long to wait to connect. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +impl Http { + fn new(url: &str, statement_timeout: Duration) -> Http { + Http { + // The client waits a little longer than the server lets a + // statement run, so the server's own timeout error arrives. + agent: ureq::AgentBuilder::new() + .timeout_connect(CONNECT_TIMEOUT) + .timeout(statement_timeout + Duration::from_secs(5)) + .build(), + url: url.trim_end_matches('/').to_string(), + max_execution_seconds: statement_timeout.as_secs().max(1).to_string(), + } + } + + /// Runs `sql` with the URL parameters `params`; its output and the + /// `read_bytes` of its summary header. The statement is the request + /// body, except for an insert, whose body is its rows. + fn run( + &self, + sql: &str, + params: &[(&str, &str)], + insert_rows: Option<&str>, + ) -> Result<(String, u64), EngineError> { + // `wait_end_of_query`: the status and the summary header come after + // the statement ends, so an error raised mid-answer is a status + // error and `read_bytes` is complete. + let mut request = self + .agent + .post(&format!("{}/", self.url)) + .query("wait_end_of_query", "1") + .query("max_execution_time", &self.max_execution_seconds); + for (name, value) in params { + request = request.query(name, value); + } + let sent = match insert_rows { + Some(rows) => request.query("query", sql).send_string(rows), + None => request.send_string(sql), + }; + match sent { + Ok(response) => { + if let Some(code) = response.header("X-ClickHouse-Exception-Code") { + let code = code.to_string(); + let text = response.into_string().unwrap_or_default(); + return Err(EngineError(format!( + "ClickHouse exception {code}: {}", + text.trim_end() + ))); + } + let read_bytes = response + .header("X-ClickHouse-Summary") + .and_then(|summary| serde_json::from_str::(summary).ok()) + .and_then(|summary| summary.get("read_bytes")?.as_str()?.parse().ok()) + .unwrap_or(0); + // `into_string` stops at 10 MB; an answer may be larger. + let mut text = String::new(); + std::io::Read::read_to_string(&mut response.into_reader(), &mut text) + .map_err(|error| EngineError(format!("reading the answer: {error}")))?; + Ok((text, read_bytes)) + } + Err(ureq::Error::Status(code, response)) => { + let text = response.into_string().unwrap_or_default(); + Err(EngineError(format!( + "ClickHouse answered {code}: {}", + text.trim_end() + ))) + } + Err(error) => Err(EngineError(format!("ClickHouse at {}: {error}", self.url))), + } + } +} diff --git a/aqpbm-engines-external/src/lib.rs b/aqpbm-engines-external/src/lib.rs index d0dada66..0a520a57 100644 --- a/aqpbm-engines-external/src/lib.rs +++ b/aqpbm-engines-external/src/lib.rs @@ -11,6 +11,7 @@ //! what each engine can run; their README gives the evidence per entry. pub mod clickhouse; +pub mod clickhouse_server; pub mod plan_file; /// A node's position in the plan file's DAG. diff --git a/aqpbm-engines-external/tests/clickhouse_conformance.rs b/aqpbm-engines-external/tests/clickhouse_conformance.rs new file mode 100644 index 00000000..d4a3837b --- /dev/null +++ b/aqpbm-engines-external/tests/clickhouse_conformance.rs @@ -0,0 +1,804 @@ +//! Conformance of the ClickHouse adapter against a real server, one case +//! per row of `capabilities/README.md` (`#[ignore]`: set +//! `AQPBM_CLICKHOUSE_URL` to the server's HTTP interface and +//! `AQPBM_CGROUP_AGENT_URL` to its cgroup agent, e.g. a server started with +//! `scripts/extbench-up.sh`). Each case installs a planner-written plan +//! through the engine, ingests hand-made chunks and compares the answers +//! with hand-computed ones: (a) rows to 1e-9 relative, (b) rows by the +//! stated difference. Run with `--test-threads 1`: the cases share the +//! server. + +mod common; + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use aqpbm_core::harness::data::{workload_chunks, DataRecord, Sample}; +use aqpbm_core::harness::engine::{ + Answers, ArmSpec, CpuAccounting, EngineDescription, EngineError, EngineUnderTest, InstalledArm, + QueryLanguage, QueryText, SampleChunk, SeriesLabels, WorkloadQuery, WorkloadSpec, +}; +use aqpbm_engines_external::clickhouse::translate; +use aqpbm_engines_external::clickhouse_server::{ClickHouseEngine, ClickHouseServer}; +use aqpbm_engines_external::plan_file::PlanFile; +use common::{fixture_path, fixture_value, fixture_value_public, lcm, payload_body}; +use serde_json::{json, Value}; + +fn server(max_threads: u32) -> ClickHouseServer { + ClickHouseServer { + url: std::env::var("AQPBM_CLICKHOUSE_URL").expect("AQPBM_CLICKHOUSE_URL"), + cgroup_agent_url: std::env::var("AQPBM_CGROUP_AGENT_URL").expect("AQPBM_CGROUP_AGENT_URL"), + max_threads, + statement_timeout: Duration::from_secs(600), + } +} + +fn engine(max_threads: u32) -> ClickHouseEngine { + ClickHouseEngine::new(server(max_threads)).expect("the server answers") +} + +/// `plan` written to a temporary file, for an arm's `plan_file`. +fn plan_file(name: &str, plan: &Value) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ch-conformance-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{name}.plan.json")); + std::fs::write(&path, plan.to_string()).unwrap(); + path +} + +fn arm(path: &std::path::Path) -> ArmSpec { + ArmSpec { + name: "conformance".into(), + raw_data_retained: true, + engine_options: json!({ "plan_file": path }), + } +} + +/// A workload of `plan`'s query texts (ids `q0`, `q1`, …), evaluated every +/// `interval_ms` (the lcm of the plan's stored steps unless given), with +/// data from 0 ms. +fn workload(plan: &Value, interval_ms: Option) -> WorkloadSpec { + let texts: Vec = serde_json::from_value(plan["query_texts"].clone()).unwrap(); + let steps = translate(&PlanFile::from_value(plan.clone()).unwrap()) + .map(|d| d.query_time_steps_ms) + .unwrap_or_default(); + let interval_ms = interval_ms.unwrap_or_else(|| steps.iter().fold(10_000, |a, &b| lcm(a, b))); + WorkloadSpec { + name: "conformance".into(), + queries: texts + .iter() + .enumerate() + .map(|(i, text)| WorkloadQuery { + id: format!("q{i}"), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: text.clone(), + }], + window_ms: 60_000, + evaluation_interval_ms: interval_ms, + }) + .collect(), + data: vec![], + } +} + +fn install(engine: &ClickHouseEngine, name: &str, plan: &Value) -> Box { + engine + .install(&workload(plan, None), &arm(&plan_file(name, plan))) + .unwrap_or_else(|e| panic!("{name}: {e}")) +} + +fn series(pairs: &[(&str, &str)]) -> Arc { + Arc::new( + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) +} + +/// `samples` cut into time-ordered chunks of `chunk_ms`, ending at +/// multiples of it, time-major as the harness's chunks are. +fn chunks(mut samples: Vec, chunk_ms: i64) -> Vec { + samples.sort_by_key(|s| s.ts_ms); + let mut out: Vec = Vec::new(); + for sample in samples { + let end_ms = (sample.ts_ms + chunk_ms - 1).div_euclid(chunk_ms) * chunk_ms; + match out.last_mut() { + Some(chunk) if chunk.end_ms == end_ms => chunk.samples.push(sample), + _ => out.push(SampleChunk { + end_ms, + samples: vec![sample], + }), + } + } + out +} + +fn ingest_all(arm: &mut dyn InstalledArm, chunks: &[SampleChunk]) { + for chunk in chunks { + arm.ingest(chunk).expect("ingest"); + } +} + +/// The answers of query `q` as (labels without `__name__` joined by +/// `,`, value), for comparing with hand-written rows. +fn rows(answers: &Answers, index: usize) -> BTreeMap { + answers + .0 + .get(&format!("q{index}")) + .into_iter() + .flatten() + .map(|(labels, value)| { + let key: Vec = labels + .iter() + .filter(|(name, _)| name.as_str() != "__name__") + .map(|(name, value)| format!("{name}={value}")) + .collect(); + (key.join(","), *value) + }) + .collect() +} + +fn assert_rows(got: &BTreeMap, expected: &[(&str, f64)]) { + assert_eq!(got.len(), expected.len(), "{got:?}"); + for (key, want) in expected { + let value = got + .get(*key) + .unwrap_or_else(|| panic!("no {key} in {got:?}")); + assert!( + (value - want).abs() <= 1e-9 * want.abs().max(1.0), + "{key}: {value} vs {want}" + ); + } +} + +#[test] +#[ignore = "needs a ClickHouse server"] +fn the_server_is_the_pinned_binary() { + let description: EngineDescription = engine(1).description(); + assert_eq!( + description.version, + "26.8.39.13 (d5bbb64d525197f84f9dd21489f5f6d6fdb966e5)" + ); + assert_eq!(description.build_profile, "max_threads=1"); +} + +/// Row "stored windows, exact Sum / Count / Min / Max" on F1's shape +/// `sum by (job) (_over_time(latency[1h]))` at t = 1 h: series a/1, a/2 +/// and b/1 with base 1.0, 0.5 and 2.0, value base · (1 + s mod 2) each +/// second. Per series over 3600 s: sum 5400 · base, count 3600, min base, +/// max 2 · base. So sum a 8100, b 10800; count a 7200, b 3600; min a 1.5, +/// b 2; max a 3, b 4. +#[test] +#[ignore = "needs a ClickHouse server"] +fn stored_sum_count_min_max() { + let engine = engine(1); + let samples: Vec = (1..=3600) + .flat_map(|s: i64| { + [("a", "1", 1.0), ("a", "2", 0.5), ("b", "1", 2.0)] + .into_iter() + .map(move |(job, i, base)| Sample { + labels: series(&[("__name__", "latency"), ("job", job), ("i", i)]), + ts_ms: s * 1000, + value: base * (1 + s % 2) as f64, + }) + }) + .collect(); + let chunks = chunks(samples, 60_000); + for (kind, expected) in [ + ("Sum", [("job=a", 8100.0), ("job=b", 10800.0)]), + ("Count", [("job=a", 7200.0), ("job=b", 3600.0)]), + ("Min", [("job=a", 1.5), ("job=b", 2.0)]), + ("Max", [("job=a", 3.0), ("job=b", 4.0)]), + ] { + let mut plan = fixture_value("sot_1h_stored_sum"); + payload_body(&mut plan, 1)["summary_type"] = json!({"ExactAggregate": [kind, kind]}); + let mut arm = install(&engine, &format!("stored_{kind}"), &plan); + ingest_all(arm.as_mut(), &chunks); + let (answers, _) = arm.evaluate(3_600_000).unwrap(); + assert_rows(&rows(&answers, 0), &expected); + let footprint = arm.footprint().unwrap(); + println!("stored {kind}: {:?} {footprint:?}", rows(&answers, 0)); + } +} + +/// The counter data of the translator tests (x, y, z, w), each series in its +/// own job, over 600 s. +fn counter_samples() -> Vec { + let mut samples = Vec::new(); + for s in 1..=600i64 { + let x = if s <= 450 { s } else { s - 450 } as f64; + let y = if s <= 360 { 3 * s } else { s - 360 } as f64; + let z = if s < 400 { + s as f64 + } else if s < 500 { + (s - 5) as f64 + } else { + (s - 500) as f64 / 2.0 + }; + for (job, value) in [("x", x), ("y", y), ("z", z)] { + samples.push(Sample { + labels: series(&[("__name__", "requests_total"), ("job", job)]), + ts_ms: s * 1000, + value, + }); + } + if s % 10 == 5 { + samples.push(Sample { + labels: series(&[("__name__", "requests_total"), ("job", "w")]), + ts_ms: s * 1000, + value: s as f64, + }); + } + } + samples +} + +/// Row "stored windows, exact Rate / Increase" (b). Three installs take the +/// 10 s chunks in turn, as `run_arm` feeds its installs, at max_threads 1 +/// and 16. Each answers PromQL's rate minus the first value after each +/// reset (rates are increase / 299 here; see the translator's +/// `stored_rate_differs_from_promql_by_the_first_value_after_each_reset`): +/// x 298/299, y 416/299, z 247/299, w 1. +#[test] +#[ignore = "needs a ClickHouse server"] +fn stored_rate_is_promql_minus_the_first_value_after_each_reset() { + let chunks = chunks(counter_samples(), 10_000); + let expected = [ + ("job=x", 298.0 / 299.0), + ("job=y", 416.0 / 299.0), + ("job=z", 247.0 / 299.0), + ("job=w", 1.0), + ]; + for threads in [1, 16] { + let engine = engine(threads); + let plan = fixture_value("rate_5m_stored"); + let mut installs: Vec<_> = (0..3) + .map(|i| install(&engine, &format!("rate_stored_{threads}_{i}"), &plan)) + .collect(); + for chunk in &chunks { + for arm in installs.iter_mut() { + arm.ingest(chunk).unwrap(); + } + } + for arm in installs.iter_mut() { + let (answers, _) = arm.evaluate(600_000).unwrap(); + assert_rows(&rows(&answers, 0), &expected); + } + assert_eq!( + installs[0].plan_summary().substitutions.len(), + 1, + "the substitute is recorded" + ); + } +} + +/// Row "query-time Rate / Increase" (a): the same data, PromQL's rates +/// x 1, y 417/299, z 642/299, w 1. +#[test] +#[ignore = "needs a ClickHouse server"] +fn query_time_rate_is_promql() { + let engine = engine(1); + let mut arm = install( + &engine, + "rate_query_time", + &fixture_value("rate_5m_query_time"), + ); + ingest_all(arm.as_mut(), &chunks(counter_samples(), 10_000)); + let (answers, _) = arm.evaluate(600_000).unwrap(); + assert_rows( + &rows(&answers, 0), + &[ + ("job=x", 1.0), + ("job=y", 417.0 / 299.0), + ("job=z", 642.0 / 299.0), + ("job=w", 1.0), + ], + ); + // Row "raw samples retained": the source table keeps rows, no view. + let footprint = arm.footprint().unwrap(); + assert!( + footprint.raw_retained_bytes > 0 && footprint.view_bytes == 0, + "{footprint:?}" + ); + println!("query-time rate footprint {footprint:?}"); +} + +/// Rows "query-time Sum … grouped" and "Aggregate": q4c at 1 m over job a +/// (1.0 and 0.5 each second) and b (2.0): at 120 s, a 90, b 120. q1 (0.99 +/// quantile per series over 1 m) of values 1..60: 1 + 0.99 · 59 = 59.41. +/// q3 (avg by job of the 0.9 quantiles) of two series 1..60 and 2·(1..60): +/// (54.1 + 108.2) / 2 = 81.15. F4's max of j3 = s over (300 s, 600 s]: 600. +#[test] +#[ignore = "needs a ClickHouse server"] +fn grouped_sums_quantiles_and_aggregates() { + let engine = engine(1); + let latency = + |rows: &[(&str, &str, f64)], seconds: std::ops::RangeInclusive, step: bool| { + seconds + .flat_map(|s| { + rows.iter().map(move |(job, id, base)| Sample { + labels: series(&[("__name__", "latency"), ("job", job), ("id", id)]), + ts_ms: s * 1000, + value: if step { base * (s - 60) as f64 } else { *base }, + }) + }) + .collect::>() + }; + let mut q4c = install( + &engine, + "q4c", + &fixture_value_public( + "q4c_sum_over_time__1m_1hz_10s", + "clickhouse.equivalent-only", + ), + ); + ingest_all( + q4c.as_mut(), + &chunks( + latency( + &[("a", "1", 1.0), ("a", "2", 0.5), ("b", "1", 2.0)], + 1..=120, + false, + ), + 10_000, + ), + ); + assert_rows( + &rows(&q4c.evaluate(120_000).unwrap().0, 0), + &[("job=a", 90.0), ("job=b", 120.0)], + ); + + // Values 1..60 over (60 s, 120 s]. + let ramp = chunks( + latency(&[("a", "1", 1.0), ("a", "2", 2.0)], 61..=120, true), + 10_000, + ); + let mut q1 = install( + &engine, + "q1", + &fixture_value_public("q1_quantile__1m_1hz_10s", "clickhouse.equivalent-only"), + ); + ingest_all(q1.as_mut(), &ramp); + let answers = q1.evaluate(120_000).unwrap().0; + assert_rows( + &rows(&answers, 0), + &[("id=1,job=a", 59.41), ("id=2,job=a", 2.0 * 59.41)], + ); + let mut q3 = install( + &engine, + "q3", + &fixture_value_public("q3_avg_quantile__1m_1hz_10s", "clickhouse.equivalent-only"), + ); + ingest_all(q3.as_mut(), &ramp); + assert_rows( + &rows(&q3.evaluate(120_000).unwrap().0, 0), + &[("job=a", 81.15)], + ); + + let mut f4 = install(&engine, "f4", &fixture_value("max_5m_label_filter")); + let samples: Vec = (1..=600) + .flat_map(|s: i64| { + [("j3", 1.0), ("j4", 10.0)] + .into_iter() + .map(move |(job, m)| Sample { + labels: series(&[("__name__", "latency"), ("job", job)]), + ts_ms: s * 1000, + value: m * s as f64, + }) + }) + .collect(); + ingest_all(f4.as_mut(), &chunks(samples, 300_000)); + assert_rows( + &rows(&f4.evaluate(600_000).unwrap().0, 0), + &[("job=j3", 600.0)], + ); +} + +/// Row "instant selector": rangeheavy's `sum by (job) (data)` with the +/// plan's 5 ms lookback at t = 120 s: s1 (a) at t − 10 ms (100) and t − 3 ms +/// (1); s2 (a) only at t − 5 ms, outside (t − 5 ms, t]; s3 (a) at t (2); s4 +/// (b) at t − 1 ms (4): a 3, b 4. +#[test] +#[ignore = "needs a ClickHouse server"] +fn instant_selector() { + let engine = engine(1); + let plan = fixture_value_public("rangeheavy", "clickhouse.equivalent-only"); + let mut arm = install(&engine, "rangeheavy", &plan); + let t = 120_000; + let samples = [ + (10, 100.0, "a", "s1"), + (5, 50.0, "a", "s2"), + (3, 1.0, "a", "s1"), + (1, 4.0, "b", "s4"), + (0, 2.0, "a", "s3"), + ] + .into_iter() + .map(|(before, value, job, id)| Sample { + labels: series(&[("__name__", "data"), ("job", job), ("id", id)]), + ts_ms: t - before, + value, + }) + .collect(); + arm.ingest(&SampleChunk { end_ms: t, samples }).unwrap(); + let answers = arm.evaluate(t).unwrap().0; + assert_rows(&rows(&answers, 0), &[("job=a", 3.0), ("job=b", 4.0)]); +} + +/// Row "Aggregate top_k": rates 1..5 for i1..i5, NaN for i6 and 5 for i7. +/// k = 2 keeps i5 and i7 (equal values: the label order breaks the tie); +/// k = 6 keeps every series but the NaN one. +#[test] +#[ignore = "needs a ClickHouse server"] +fn top_k_ties_and_nan() { + let engine = engine(1); + let samples: Vec = (1..=600) + .flat_map(|s: i64| { + [1.0, 2.0, 3.0, 4.0, 5.0, f64::NAN, 5.0] + .into_iter() + .enumerate() + .map(move |(i, rate)| Sample { + labels: series(&[ + ("__name__", "requests_total"), + ("instance", &format!("i{}", i + 1)), + ]), + ts_ms: s * 1000, + value: rate * s as f64, + }) + }) + .collect(); + let chunks = chunks(samples, 60_000); + for (k, expected) in [ + (2, vec![("instance=i5", 5.0), ("instance=i7", 5.0)]), + ( + 6, + vec![ + ("instance=i1", 1.0), + ("instance=i2", 2.0), + ("instance=i3", 3.0), + ("instance=i4", 4.0), + ("instance=i5", 5.0), + ("instance=i7", 5.0), + ], + ), + ] { + let mut plan = fixture_value_public("topk__5m__1hz__60s", "clickhouse.equivalent-only"); + payload_body(&mut plan, 5)["measures"][0]["k"] = json!(k); + let mut arm = install(&engine, &format!("topk_{k}"), &plan); + ingest_all(arm.as_mut(), &chunks); + assert_rows(&rows(&arm.evaluate(600_000).unwrap().0, 0), &expected); + } +} + +/// Rows "query-time … regrouping merges" and "Project / BinaryOp": +/// mixed-03 root 11, `avg by (region, service) (quantile_over_time(0.9, +/// data[100s]))` as sum / count, at 120 s over constant series (j1, r1) 1, +/// (j2, r1) 3, (j1, r2) 5: r1/s1 2, r2/s1 5. Two p90s of 1.5e308 overflow +/// the sum: the checked division is an error. +#[test] +#[ignore = "needs a ClickHouse server"] +fn average_as_sum_over_count_and_checked_division() { + let engine = engine(1); + let plan = fixture_value_public("mixed-03", "clickhouse.equivalent-only"); + let constant = |rows: &[(&str, &str, f64)]| { + (1..=120) + .flat_map(|s: i64| { + rows.iter().map(move |(job, region, value)| Sample { + labels: series(&[ + ("__name__", "data"), + ("job", job), + ("region", region), + ("service", "s1"), + ]), + ts_ms: s * 1000, + value: *value, + }) + }) + .collect::>() + }; + let mut arm = install(&engine, "mixed03", &plan); + ingest_all( + arm.as_mut(), + &chunks( + constant(&[("j1", "r1", 1.0), ("j2", "r1", 3.0), ("j1", "r2", 5.0)]), + 30_000, + ), + ); + let answers = arm.evaluate(120_000).unwrap().0; + assert_rows( + &rows(&answers, 3), + &[("region=r1,service=s1", 2.0), ("region=r2,service=s1", 5.0)], + ); + let mut overflow = install(&engine, "mixed03_overflow", &plan); + ingest_all( + overflow.as_mut(), + &chunks( + constant(&[("j1", "r1", 1.5e308), ("j2", "r1", 1.5e308)]), + 30_000, + ), + ); + let error = overflow.evaluate(120_000).expect_err("an error"); + assert!(error.0.contains("checked division"), "{error}"); +} + +/// Rows of class (c): IRate and a query-time result kept across +/// evaluations are refused at install. +#[test] +#[ignore = "needs a ClickHouse server"] +fn class_c_rows_are_refused() { + let engine = engine(1); + let mut irate = fixture_value("rate_5m_query_time"); + payload_body(&mut irate, 2)["summary_type"] = json!({"ExactAggregate": ["IRate", "IRate"]}); + let error = engine + .install(&workload(&irate, None), &arm(&plan_file("irate", &irate))) + .err() + .expect("refused"); + assert!(error.0.contains("exact IRate"), "{error}"); + let mut kept = fixture_value("joint_5m"); + kept["dag"]["nodes"][1]["lifetime"] = json!({"AcrossEvaluations": "Memory"}); + let error = engine + .install(&workload(&kept, None), &arm(&plan_file("kept", &kept))) + .err() + .expect("refused"); + assert!(error.0.contains("kept across evaluations"), "{error}"); +} + +/// Rows "raw samples retained" (a stored-only metric keeps no rows) and +/// "retention" (the TTL statements apply on the server). +#[test] +#[ignore = "needs a ClickHouse server"] +fn storage_and_retention() { + let engine = engine(1); + let plan = fixture_value("sot_1h_stored_sum"); + let mut arm = install(&engine, "sot_storage", &plan); + let samples: Vec = (1..=120) + .map(|s: i64| Sample { + labels: series(&[("__name__", "latency"), ("job", "a")]), + ts_ms: s * 1000, + value: 1.0, + }) + .collect(); + ingest_all(arm.as_mut(), &chunks(samples, 60_000)); + let footprint = arm.footprint().unwrap(); + assert!( + footprint.raw_retained_bytes == 0 && footprint.view_bytes > 0, + "{footprint:?}" + ); + println!("stored-only footprint {footprint:?}"); + + let deployment = translate(&PlanFile::from_value(plan).unwrap()).unwrap(); + let url = server(1).url; + let http = |sql: &str| { + ureq::post(&format!("{url}/")) + .query("database", "conformance_ttl") + .send_string(sql) + .map(|r| r.into_string().unwrap()) + .map_err(|e| e.to_string()) + }; + ureq::post(&format!("{url}/")) + .send_string("CREATE DATABASE IF NOT EXISTS conformance_ttl") + .unwrap(); + for statement in deployment.tables.iter().chain(&deployment.ttl_statements()) { + http(statement).unwrap_or_else(|e| panic!("{statement}: {e}")); + } + let created = http("SELECT create_table_query FROM system.tables WHERE database = 'conformance_ttl' AND name = 'v1'").unwrap(); + assert!(created.contains("TTL"), "{created}"); + ureq::post(&format!("{url}/")) + .send_string("DROP DATABASE conformance_ttl SYNC") + .unwrap(); +} + +/// The harness kit on an engine that names the planner-written probe plans +/// (`sum_over_time(probe[10s])`, `count_over_time(probe[10s])`): answers +/// are deterministic and windows are left-open. +#[test] +#[ignore = "needs a ClickHouse server"] +fn the_harness_conformance_kit_passes() { + struct WithProbePlans(ClickHouseEngine); + impl EngineUnderTest for WithProbePlans { + fn description(&self) -> EngineDescription { + self.0.description() + } + fn cpu_accounting(&self) -> CpuAccounting { + self.0.cpu_accounting() + } + fn install( + &self, + workload: &WorkloadSpec, + arm: &ArmSpec, + ) -> Result, EngineError> { + let plan = match workload.name.as_str() { + "probe-sum" => "probe_sum_10s", + "probe-count" => "probe_count_10s", + other => return Err(EngineError(format!("no probe plan for {other}"))), + }; + let mut arm = arm.clone(); + arm.engine_options = json!({ "plan_file": fixture_path(plan) }); + self.0.install(workload, &arm) + } + } + let mut engine = WithProbePlans(engine(1)); + let clock = aqpbm_core::harness::protocol::OsCpuClock; + let clocks = aqpbm_core::harness::protocol::MeasurementClocks::with_calling_thread(&clock); + let report = aqpbm_core::harness::conformance::check(&mut engine, &clocks); + println!("harness kit: {report:?}"); + let passed = report.expect("the kit passes").passed; + assert!( + passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::DeterministicAnswers) + ); + assert!(passed.contains(&aqpbm_core::harness::conformance::ConformanceCheck::LeftOpenWindow)); +} + +/// Suite size: 1,000 series in 10 jobs, 1 s samples for 10 min from +/// 1,700,000,000,000 ms, q4c at 1 m evaluated every 10 s. Every chunk is one +/// insert of 10,000 rows; at the last evaluation each job's answer equals +/// the sum, computed here, of its samples in the last minute. The server +/// cgroup's CPU grows over the ingest and is never throttled. +#[test] +#[ignore = "needs a ClickHouse server"] +fn suite_sized_workload() { + let engine = engine(1); + let start_ms: i64 = 1_700_000_000_000; + let record: DataRecord = serde_json::from_value(json!({ + "metric": "latency", + "description": { + "labels": [{"name": "job", "cardinality": 10}, {"name": "instance", "cardinality": 100}], + "series_num": 1000, "start_ms": start_ms, "sample_interval_ms": 1000, + "duration_ms": 600000, + "value_distribution": {"kind": "uniform", "lower_bound": 0.0, "upper_bound": 10.0, "seed": 0}, + "counter": false, "seed": 7 + } + })) + .unwrap(); + let plan = fixture_value_public( + "q4c_sum_over_time__1m_1hz_10s", + "clickhouse.equivalent-only", + ); + let mut spec = workload(&plan, Some(10_000)); + spec.data = vec![record.clone()]; + let mut arm = engine + .install(&spec, &arm(&plan_file("suite", &plan))) + .expect("installs"); + let CpuAccounting::Cgroup(source) = engine.cpu_accounting() else { + panic!("cgroup accounting"); + }; + let cpu_before = source.cpu().expect("the agent answers"); + let mut last_minute: BTreeMap = BTreeMap::new(); + let end_ms = start_ms + 600_000; + let ingest_started = std::time::Instant::now(); + let mut rows_per_chunk = Vec::new(); + for chunk in workload_chunks(&[record], 10_000).unwrap() { + let chunk = chunk.unwrap(); + rows_per_chunk.push(chunk.samples.len()); + for sample in &chunk.samples { + if sample.ts_ms > end_ms - 60_000 { + *last_minute.entry(sample.labels["job"].clone()).or_default() += sample.value; + } + } + arm.ingest(&chunk).unwrap(); + if chunk.end_ms >= start_ms + 60_000 { + arm.evaluate(chunk.end_ms).unwrap(); + } + } + let ingest_wall = ingest_started.elapsed(); + let cpu_after = source.cpu().unwrap(); + let answers = arm.evaluate(end_ms).unwrap().0; + let got = rows(&answers, 0); + let expected: Vec<(String, f64)> = last_minute + .iter() + .map(|(job, sum)| (format!("job={job}"), *sum)) + .collect(); + let expected: Vec<(&str, f64)> = expected.iter().map(|(k, v)| (k.as_str(), *v)).collect(); + assert_rows(&got, &expected); + assert!( + rows_per_chunk.iter().all(|&n| n == 10_000), + "{rows_per_chunk:?}" + ); + assert!(cpu_after.usage_ns > cpu_before.usage_ns); + assert_eq!(cpu_after.nr_throttled, cpu_before.nr_throttled); + println!( + "suite size: 60 chunks of 10,000 rows, 55 evaluations; wall {:.1} s; server cgroup CPU {:.3} s", + ingest_wall.as_secs_f64(), + (cpu_after.usage_ns - cpu_before.usage_ns) as f64 / 1e9 + ); + // The idle server's CPU over 10 s. + let idle_before = source.cpu().unwrap(); + std::thread::sleep(Duration::from_secs(10)); + let idle_after = source.cpu().unwrap(); + println!( + "idle server cgroup CPU over 10 s: {:.1} ms", + (idle_after.usage_ns - idle_before.usage_ns) as f64 / 1e6 + ); +} + +/// The lane end to end: the probe workloads on lane `fake` and lane +/// `clickhouse` over one run directory, reported with the fake as truth: +/// every point is there, and none differs. +#[test] +#[ignore = "needs a ClickHouse server"] +fn the_clickhouse_lane_matches_the_fake_lane() { + use aqpbm_core::harness::bench_external::{report, run_lane, LaneSetup}; + use aqpbm_core::harness::fake::FakeEngine; + use aqpbm_core::harness::protocol::{ + load_probe_for, MeasurementClocks, OsCpuClock, ProcessCpuFiles, Protocol, + }; + use aqpbm_core::harness::rundir::RunDir; + + let root = std::env::temp_dir().join(format!("ch-lane-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let run_dir = RunDir::create_at(&root).unwrap(); + let write = |path: PathBuf, text: String| { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); + }; + for workload in aqpbm_core::harness::conformance::probe_workloads() { + let plan = match workload.name.as_str() { + "probe-sum" => "probe_sum_10s", + _ => "probe_count_10s", + }; + write( + run_dir + .path() + .join(format!("workloads/{}.json", workload.name)), + serde_json::to_string(&workload).unwrap(), + ); + write( + run_dir + .path() + .join(format!("plans/equivalent-only/{}.plan.json", workload.name)), + std::fs::read_to_string(fixture_path(plan)).unwrap(), + ); + } + write( + run_dir.path().join("plans/equivalent-only/variant.json"), + r#"{"substitutions_allowed": false}"#.into(), + ); + let protocol = Protocol { + warmup_runs: 1, + measured_runs: 3, + max_core_busy_fraction: 1.0, + }; + let clock = OsCpuClock; + let clocks = MeasurementClocks::with_calling_thread(&clock); + let fake = FakeEngine::default(); + let clickhouse = engine(1); + for lane in [&fake as &dyn EngineUnderTest, &clickhouse] { + let mut probe = load_probe_for(&lane.cpu_accounting(), &ProcessCpuFiles::os()).unwrap(); + let cells = run_lane( + &run_dir, + lane, + None, + LaneSetup { + protocol: &protocol, + core: 0, + probe: probe.as_mut(), + clocks: &clocks, + args: vec![], + }, + ) + .unwrap(); + println!("{} cells: {cells:?}", lane.description().id); + } + let report = report(&run_dir, "fake", "equivalent-only").unwrap(); + println!("{}", report.summary_markdown()); + let csv = report.summary_csv(); + let clickhouse_rows: Vec<&str> = csv.lines().filter(|l| l.contains(",clickhouse,")).collect(); + assert_eq!(clickhouse_rows.len(), 2, "{csv}"); + let accuracy: Vec<_> = report + .accuracy + .iter() + .filter(|a| a.lane == "clickhouse") + .collect(); + assert_eq!(accuracy.len(), 2, "{:?}", report.accuracy); + for query in accuracy { + let errors = &query.errors; + assert!(errors.points > 0, "{query:?}"); + assert_eq!(errors.missing_points, 0, "{query:?}"); + assert_eq!(errors.extra_points, 0, "{query:?}"); + assert!(errors.max_rel_error.unwrap_or(0.0) <= 1e-9, "{query:?}"); + } + let _ = std::fs::remove_dir_all(&root); +} diff --git a/aqpbm-engines-external/tests/clickhouse_server.rs b/aqpbm-engines-external/tests/clickhouse_server.rs new file mode 100644 index 00000000..8cb93cdc --- /dev/null +++ b/aqpbm-engines-external/tests/clickhouse_server.rs @@ -0,0 +1,731 @@ +//! S1–S10: the ClickHouse engine adapter against an in-test HTTP stub that +//! records every request and answers by rules. No server is needed. + +mod common; + +use std::collections::BTreeMap; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use aqpbm_core::harness::cgroup::CgroupSource; +use aqpbm_core::harness::data::{DataRecord, Sample}; +use aqpbm_core::harness::engine::{ + ArmSpec, CpuAccounting, EngineUnderTest, QueryLanguage, QueryText, SampleChunk, SeriesLabels, + WorkloadQuery, WorkloadSpec, +}; +use aqpbm_engines_external::clickhouse_server::{ + series_identity, ClickHouseEngine, ClickHouseServer, +}; +use common::{fixture_path, fixture_value, payload_body}; +use serde_json::json; + +/// One request the stub received. +#[derive(Debug, Clone)] +struct Request { + params: BTreeMap, + body: String, +} + +impl Request { + /// The statement: the `query` parameter of an insert, else the body. + fn statement(&self) -> &str { + self.params + .get("query") + .map(String::as_str) + .unwrap_or(&self.body) + } +} + +/// An answer: status, extra headers, body. +type Reply = (u16, Vec<(String, String)>, String); +type Rule = dyn Fn(&Request) -> Option + Send + Sync; + +/// The stub: answers the version and leftover-database questions, then the +/// first rule that matches, else `200 ""`. +struct Stub { + url: String, + requests: Arc>>, +} + +fn start_stub(rules: Vec>) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = Arc::clone(&requests); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { return }; + let Some(request) = read_request(&mut stream) else { + continue; + }; + seen.lock().unwrap().push(request.clone()); + let (status, headers, body) = rules + .iter() + .find_map(|rule| rule(&request)) + .or_else(|| default_reply(&request)) + .unwrap_or((200, vec![], String::new())); + let mut head = format!( + "HTTP/1.1 {status} X\r\nContent-Length: {}\r\nConnection: close\r\n", + body.len() + ); + for (name, value) in headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + let _ = stream.write_all(format!("{head}\r\n{body}").as_bytes()); + } + }); + Stub { url, requests } +} + +fn default_reply(request: &Request) -> Option { + let statement = request.statement(); + if statement.contains("SELECT version()") { + return Some((200, vec![], "26.8.39.13\td5bbb64d\n".into())); + } + if statement.contains("system.databases") { + return Some((200, vec![], String::new())); + } + None +} + +fn read_request(stream: &mut std::net::TcpStream) -> Option { + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).ok()?; + let target = line.split_whitespace().nth(1)?.to_string(); + let mut length = 0; + loop { + let mut header = String::new(); + reader.read_line(&mut header).ok()?; + if header == "\r\n" || header.is_empty() { + break; + } + if let Some((name, value)) = header.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + length = value.trim().parse().ok()?; + } + } + } + let mut body = vec![0; length]; + reader.read_exact(&mut body).ok()?; + let params = target + .split_once('?') + .map(|(_, query)| query) + .unwrap_or("") + .split('&') + .filter(|pair| !pair.is_empty()) + .map(|pair| { + let (name, value) = pair.split_once('=').unwrap_or((pair, "")); + (decode(name), decode(value)) + }) + .collect(); + Some(Request { + params, + body: String::from_utf8(body).ok()?, + }) +} + +/// URL form decoding: `+` is a space, `%XX` a byte. +fn decode(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'+' => out.push(b' '), + b'%' if i + 2 < bytes.len() => { + out.push(u8::from_str_radix(&text[i + 1..i + 3], 16).unwrap()); + i += 2; + } + b => out.push(b), + } + i += 1; + } + String::from_utf8(out).unwrap() +} + +fn engine(stub: &Stub) -> ClickHouseEngine { + ClickHouseEngine::new(ClickHouseServer { + url: stub.url.clone(), + cgroup_agent_url: "http://10.0.0.2:9101".into(), + max_threads: 1, + statement_timeout: Duration::from_secs(5), + }) + .expect("engine") +} + +fn arm(plan: &str) -> ArmSpec { + ArmSpec { + name: "equivalent-only".into(), + raw_data_retained: true, + engine_options: json!({ "plan_file": fixture_path(plan) }), + } +} + +/// A workload whose queries are the plan's texts, with ids `q0`, `q1`, …, +/// evaluated every `interval_ms`, with data from `start_ms`. +fn workload_of(plan: &str, interval_ms: u64, start_ms: i64) -> WorkloadSpec { + let value = fixture_value(plan); + let texts: Vec = serde_json::from_value(value["query_texts"].clone()).unwrap(); + WorkloadSpec { + name: plan.into(), + queries: texts + .iter() + .enumerate() + .map(|(i, text)| WorkloadQuery { + id: format!("q{i}"), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: text.clone(), + }], + window_ms: 60_000, + evaluation_interval_ms: interval_ms, + }) + .collect(), + data: vec![data_record(start_ms)], + } +} + +fn data_record(start_ms: i64) -> DataRecord { + serde_json::from_value(json!({ + "metric": "latency", + "description": { + "labels": [], "series_num": 1, "start_ms": start_ms, "sample_interval_ms": 1000, + "duration_ms": 60000, + "value_distribution": {"kind": "uniform", "lower_bound": 0.0, "upper_bound": 1.0, "seed": 0}, + "counter": false, "seed": 1 + } + })) + .unwrap() +} + +fn labels(pairs: &[(&str, &str)]) -> Arc { + Arc::new( + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) +} + +fn statements(stub: &Stub) -> Vec { + stub.requests + .lock() + .unwrap() + .iter() + .map(|r| r.statement().to_string()) + .collect() +} + +#[test] +fn install_creates_a_database_and_the_translated_tables() { + // S1 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let installed = engine + .install( + &workload_of("rate_5m_stored", 60_000, 0), + &arm("rate_5m_stored"), + ) + .expect("installs"); + let requests = stub.requests.lock().unwrap().clone(); + // version, leftover check, CREATE DATABASE, src table, view table, view. + let created: Vec<&Request> = requests + .iter() + .filter(|r| r.statement().starts_with("CREATE")) + .collect(); + assert_eq!(created.len(), 4, "{requests:#?}"); + let database = created[0] + .statement() + .strip_prefix("CREATE DATABASE ") + .unwrap() + .to_string(); + assert!(database.starts_with("bench_")); + for request in &created[1..] { + assert_eq!(request.params.get("database"), Some(&database)); + assert_eq!( + request.params.get("wait_end_of_query").map(String::as_str), + Some("1") + ); + } + assert!(created[1] + .statement() + .starts_with("CREATE TABLE src_requests_total")); + assert!(created[2].statement().starts_with("CREATE TABLE v1")); + assert!(created[3] + .statement() + .starts_with("CREATE MATERIALIZED VIEW v1_mv")); + let summary = installed.plan_summary(); + assert_eq!( + summary.substitutions, + vec![ + "clickhouse.stored_rate.delta_sum_timestamp at node 1: a counter reset adds 0 \ + for the step across it, where PromQL adds the first value after the reset" + ] + ); + assert_eq!(summary.stored_outputs, vec!["v1"]); + assert_eq!(summary.node_kinds[1], "SummaryTumblingWindowChain"); + // S8: dropping the arm drops its database, synchronously. + drop(installed); + assert!(statements(&stub).contains(&format!("DROP DATABASE IF EXISTS {database} SYNC"))); + + let joint = engine + .install(&workload_of("joint_5m", 60_000, 0), &arm("joint_5m")) + .expect("installs"); + assert_eq!( + joint.plan_summary().sharing_lost, + vec!["node 1 for nodes 2, 3, 4"] + ); +} + +#[test] +fn install_maps_queries_by_index_and_text() { + // S2 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let mut workload = workload_of("sot_1h_stored_sum", 60_000, 0); + // Another language's text that equals the plan's is accepted. + workload.queries[0].texts.insert( + 0, + QueryText { + language: QueryLanguage::Sql, + text: "SELECT 1".into(), + }, + ); + assert!(engine.install(&workload, &arm("sot_1h_stored_sum")).is_ok()); + workload.queries[0].texts.remove(1); + let error = engine + .install(&workload, &arm("sot_1h_stored_sum")) + .err() + .expect("refused"); + assert!( + error + .0 + .contains("sum by (job) (sum_over_time(latency[1h]))") + && error.0.contains("SELECT 1"), + "{error}" + ); + let mut two = workload_of("sot_1h_stored_sum", 60_000, 0); + two.queries.push(two.queries[0].clone()); + assert!(engine + .install(&two, &arm("sot_1h_stored_sum")) + .err() + .expect("refused") + .0 + .contains("1 queries")); + // A stored step (60 s) that does not divide the evaluation interval. + let error = engine + .install( + &workload_of("sot_1h_stored_sum", 90_000, 0), + &arm("sot_1h_stored_sum"), + ) + .err() + .expect("refused"); + assert!(error.0.contains("60000 ms"), "{error}"); + // ... or the data start. + assert!(engine + .install( + &workload_of("sot_1h_stored_sum", 60_000, 30_000), + &arm("sot_1h_stored_sum") + ) + .is_err()); +} + +#[test] +fn install_refuses_missing_unreadable_and_untranslatable_plans() { + // S3 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let workload = workload_of("sot_1h_stored_sum", 60_000, 0); + let mut no_plan = arm("sot_1h_stored_sum"); + no_plan.engine_options = json!({}); + assert!(engine + .install(&workload, &no_plan) + .err() + .unwrap() + .0 + .contains("plan_file")); + let mut missing = arm("sot_1h_stored_sum"); + missing.engine_options = json!({"plan_file": "/nonexistent/plan.json"}); + assert!(engine + .install(&workload, &missing) + .err() + .unwrap() + .0 + .contains("/nonexistent")); + let mut value = fixture_value("sot_1h_stored_sum"); + payload_body(&mut value, 1)["summary_type"] = json!({"ExactAggregate": ["IRate", "IRate"]}); + let dir = std::env::temp_dir().join(format!("ch-server-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("irate.plan.json"); + std::fs::write(&path, value.to_string()).unwrap(); + let mut irate = arm("sot_1h_stored_sum"); + irate.engine_options = json!({ "plan_file": path }); + let error = engine.install(&workload, &irate).err().expect("refused"); + assert!( + error.0.contains("no translation for exact IRate"), + "{error}" + ); + // Nothing was created for a refused install. + assert!(!statements(&stub).iter().any(|s| s.starts_with("CREATE"))); +} + +#[test] +fn ingest_inserts_each_metric_in_chunk_order() { + // S4 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let mut installed = engine + .install( + &workload_of("max_5m_label_filter", 300_000, 0), + &arm("max_5m_label_filter"), + ) + .unwrap(); + let odd = "a\tb\nc\\d\"é"; + let a = labels(&[("__name__", "latency"), ("job", "j3")]); + let b = labels(&[("__name__", "latency"), ("instance", odd)]); + let other = labels(&[("__name__", "unread")]); + let chunk = SampleChunk { + end_ms: 2000, + samples: vec![ + Sample { + labels: Arc::clone(&a), + ts_ms: 1000, + value: 1.5, + }, + Sample { + labels: Arc::clone(&b), + ts_ms: 1000, + value: f64::NAN, + }, + Sample { + labels: other, + ts_ms: 1000, + value: 9.0, + }, + Sample { + labels: Arc::clone(&a), + ts_ms: 2000, + value: 2.0, + }, + ], + }; + let report = installed.ingest(&chunk).unwrap(); + assert_eq!(report.engine_reported_cpu_ns, None); + let requests = stub.requests.lock().unwrap().clone(); + let inserts: Vec<&Request> = requests + .iter() + .filter(|r| r.statement().starts_with("INSERT")) + .collect(); + assert_eq!( + inserts.len(), + 1, + "one insert for the one metric the plan reads" + ); + let insert = inserts[0]; + assert!( + insert.statement().starts_with( + "INSERT INTO src_latency (ts, value, `job`, `$promql_series_identity`) \ + SETTINGS input_format_null_as_default = 0, insert_null_as_default = 0 SELECT" + ), + "{}", + insert.statement() + ); + let escaped_identity = |l: &SeriesLabels| { + series_identity(l) + .replace('\\', "\\\\") + .replace('\t', "\\t") + .replace('\n', "\\n") + }; + assert_eq!( + insert.body, + format!( + "1000\t1.5\tj3\t{}\n1000\tnan\t\t{}\n2000\t2\tj3\t{}\n", + escaped_identity(&a), + escaped_identity(&b), + escaped_identity(&a) + ) + ); + assert_eq!(series_identity(&a), r#"{"__name__":"latency","job":"j3"}"#); + // A sample at 0 ms is refused (the window key needs ts > 0). + let zero = SampleChunk { + end_ms: 0, + samples: vec![Sample { + labels: a, + ts_ms: 0, + value: 1.0, + }], + }; + assert!(installed.ingest(&zero).is_err()); +} + +#[test] +fn evaluate_parses_series_and_values() { + // S5, S6, S6b + let identity = r#"{"__name__":"latency","instance":"a\tb","job":"j3"}"#; + let escaped = identity.replace('\\', "\\\\").replace('\t', "\\t"); + let answer = format!("job\t$promql_series_identity\tvalue\nj3\t{escaped}\tnan\n"); + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| { + ( + 200, + vec![( + "X-ClickHouse-Summary".to_string(), + r#"{"read_rows":"3","read_bytes":"48"}"#.to_string(), + )], + answer.clone(), + ) + }) + })]); + let engine = engine(&stub); + let mut installed = engine + .install( + &workload_of("max_5m_label_filter", 300_000, 0), + &arm("max_5m_label_filter"), + ) + .unwrap(); + let (answers, report) = installed.evaluate(600_000).unwrap(); + // The root applies a function (max_over_time), so the answer drops + // `__name__`, as PromQL's output does. + let mut series: SeriesLabels = serde_json::from_str(identity).unwrap(); + series.remove("__name__"); + assert!(answers.get("q0", &series).unwrap().is_nan(), "{answers:?}"); + assert_eq!(report.scanned_bytes, Some(48)); + assert_eq!(report.engine_reported_cpu_ns, None); + let query = stub + .requests + .lock() + .unwrap() + .iter() + .rev() + .find(|r| r.statement().contains("FORMAT TabSeparatedWithNames")) + .cloned() + .unwrap(); + assert_eq!( + query.params.get("param_t").map(String::as_str), + Some("600000") + ); + assert_eq!( + query.params.get("max_threads").map(String::as_str), + Some("1") + ); + assert!(query.params.get("database").unwrap().starts_with("bench_")); + + // Without an identity column: the label columns, empty ones left out; + // two rows of one series are an error. + for (body, expected) in [ + ( + "job\tvalue\nj3\t2\n\t5\n", + Ok(vec![(vec![("job", "j3")], 2.0), (vec![], 5.0)]), + ), + ("job\tvalue\nj3\t2\nj3\t5\n", Err("two rows")), + ] { + let body = body.to_string(); + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| (200, vec![], body.clone())) + })]); + let mut installed = engine_for(&stub); + match (installed.evaluate(600_000), expected) { + (Ok((answers, _)), Ok(rows)) => { + for (pairs, value) in rows { + let key: SeriesLabels = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + assert_eq!(answers.get("q0", &key), Some(value)); + } + } + (Err(error), Err(text)) => assert!(error.0.contains(text), "{error}"), + (got, want) => panic!("{got:?} vs {want:?}"), + } + } + + // S6: an error status; S6b: status 200 with an exception header. + for reply in [ + ( + 500, + vec![], + "Code: 395. DB::Exception: not a window end".to_string(), + ), + ( + 200, + vec![("X-ClickHouse-Exception-Code".to_string(), "395".to_string())], + "Code: 395. DB::Exception: not a window end".to_string(), + ), + ] { + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| reply.clone()) + })]); + let error = engine_for(&stub).evaluate(600_000).expect_err("an error"); + assert!(error.0.contains("not a window end"), "{error}"); + } +} + +fn engine_for(stub: &Stub) -> Box { + engine(stub) + .install( + &workload_of("max_5m_label_filter", 300_000, 0), + &arm("max_5m_label_filter"), + ) + .unwrap() +} + +#[test] +fn a_server_that_never_answers_times_out() { + // S6c: a listener that accepts and never answers. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let _held: Vec<_> = listener.incoming().take(4).collect(); + std::thread::sleep(Duration::from_secs(60)); + }); + let started = std::time::Instant::now(); + let result = ClickHouseEngine::new(ClickHouseServer { + url, + cgroup_agent_url: "http://10.0.0.2:9101".into(), + max_threads: 1, + statement_timeout: Duration::from_secs(1), + }); + assert!(result.is_err()); + assert!(started.elapsed() < Duration::from_secs(20)); +} + +#[test] +fn footprint_splits_view_and_raw_bytes() { + // S7 + let stub = start_stub(vec![Box::new(|request: &Request| { + request + .statement() + .contains("system.parts") + .then(|| (200, vec![], "src_latency\t100\nv1\t40\n".to_string())) + })]); + let engine = engine(&stub); + let installed = engine + .install( + &workload_of("sot_1h_stored_sum", 60_000, 0), + &arm("sot_1h_stored_sum"), + ) + .unwrap(); + let footprint = installed.footprint().unwrap(); + assert_eq!(footprint.view_bytes, 40); + assert_eq!(footprint.raw_retained_bytes, 100); +} + +#[test] +fn description_and_cpu_accounting() { + // S9 + let stub = start_stub(vec![]); + let engine = engine(&stub); + let description = engine.description(); + assert_eq!(description.id, "clickhouse"); + assert_eq!(description.version, "26.8.39.13 (d5bbb64d)"); + assert_eq!(description.build_profile, "max_threads=1"); + assert_eq!( + engine.cpu_accounting(), + CpuAccounting::Cgroup(CgroupSource::Agent { + url: "http://10.0.0.2:9101".into() + }) + ); +} + +#[test] +fn leftover_databases_are_refused() { + // S10 + let stub = start_stub(vec![Box::new(|request: &Request| { + request + .statement() + .contains("system.databases") + .then(|| (200, vec![], "bench_1_0\nbench_1_1\n".to_string())) + })]); + let error = ClickHouseEngine::new(ClickHouseServer { + url: stub.url.clone(), + cgroup_agent_url: "http://10.0.0.2:9101".into(), + max_threads: 1, + statement_timeout: Duration::from_secs(5), + }) + .err() + .expect("refused"); + assert!(error.0.contains("bench_1_0, bench_1_1"), "{error}"); +} + +#[test] +fn a_failed_install_drops_its_database() { + // The database exists once CREATE DATABASE ran; a table that fails + // drops it again. + let stub = start_stub(vec![Box::new(|request: &Request| { + request.statement().starts_with("CREATE TABLE").then(|| { + ( + 500, + vec![], + "Code: 57. DB::Exception: table exists".to_string(), + ) + }) + })]); + let engine = engine(&stub); + let error = engine + .install( + &workload_of("sot_1h_stored_sum", 60_000, 0), + &arm("sot_1h_stored_sum"), + ) + .err() + .expect("refused"); + assert!(error.0.contains("table exists"), "{error}"); + let statements = statements(&stub); + let database = statements + .iter() + .find_map(|s| s.strip_prefix("CREATE DATABASE ")) + .expect("a database was created") + .to_string(); + assert!(statements.contains(&format!("DROP DATABASE IF EXISTS {database} SYNC"))); +} + +#[test] +fn only_a_bare_selector_keeps_the_metric_name() { + // rangeheavy node 1 is the instant selector `data` (5 ms lookback): a + // root that is only that selector keeps `__name__`, as PromQL does. + let identity = r#"{"__name__":"data","job":"a"}"#; + let answer = format!("job\t$promql_series_identity\tvalue\na\t{identity}\t3\n"); + let stub = start_stub(vec![Box::new(move |request: &Request| { + request + .statement() + .contains("FORMAT TabSeparatedWithNames") + .then(|| (200, vec![], answer.clone())) + })]); + let mut plan = common::fixture_value_public("rangeheavy", "clickhouse.equivalent-only"); + plan["dag"]["roots"] = json!([1]); + plan["query_texts"] = json!(["data"]); + let dir = std::env::temp_dir().join(format!("ch-server-selector-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("selector.plan.json"); + std::fs::write(&path, plan.to_string()).unwrap(); + let workload = WorkloadSpec { + name: "selector".into(), + queries: vec![WorkloadQuery { + id: "q0".into(), + texts: vec![QueryText { + language: QueryLanguage::PromQl, + text: "data".into(), + }], + window_ms: 60_000, + evaluation_interval_ms: 60_000, + }], + data: vec![data_record(0)], + }; + let mut selector_arm = arm("sot_1h_stored_sum"); + selector_arm.engine_options = json!({ "plan_file": path }); + let mut installed = engine(&stub).install(&workload, &selector_arm).unwrap(); + let (answers, _) = installed.evaluate(60_000).unwrap(); + let series: SeriesLabels = serde_json::from_str(identity).unwrap(); + assert_eq!(answers.get("q0", &series), Some(3.0), "{answers:?}"); +} diff --git a/aqpbm-engines-external/tests/extbench_script.rs b/aqpbm-engines-external/tests/extbench_script.rs new file mode 100644 index 00000000..510c48bc --- /dev/null +++ b/aqpbm-engines-external/tests/extbench_script.rs @@ -0,0 +1,73 @@ +//! `scripts/extbench-up.sh`: it parses, and it refuses to listen on an +//! address that is not private (the server has no password). + +use std::process::Command; + +fn script() -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../scripts/extbench-up.sh") +} + +#[test] +fn the_script_parses() { + let out = Command::new("bash") + .arg("-n") + .arg(script()) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); +} + +#[test] +fn only_private_listen_addresses_are_accepted() { + for (address, accepted) in [ + ("10.0.0.2", true), + ("172.16.4.1", true), + ("192.168.1.9", true), + ("127.0.0.1", true), + ("0.0.0.0", false), + ("8.8.8.8", false), + ("172.32.0.1", false), + ("10.0.0", false), + ("10.0.0.256", false), + ] { + let out = Command::new("bash") + .arg(script()) + .args(["check-listen-host", address]) + .output() + .unwrap(); + assert_eq!(out.status.success(), accepted, "{address}"); + } +} + +#[test] +fn a_data_dir_the_script_did_not_make_is_never_wiped() { + let dir = std::env::temp_dir().join(format!("extbench-guard-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("keep"), "data").unwrap(); + let binary = "/bin/true"; + let sha256 = Command::new("sha256sum").arg(binary).output().unwrap(); + let sha256 = String::from_utf8(sha256.stdout).unwrap(); + let out = Command::new("bash") + .arg(script()) + .args(["clickhouse", "up"]) + .arg(&dir) + .args(["2,22", "0", "18123", "19101"]) + .env("LISTEN_HOST", "127.0.0.1") + .env("CLICKHOUSE_BINARY", binary) + .env( + "CLICKHOUSE_SHA256", + sha256.split_whitespace().next().unwrap(), + ) + .env("BASE_IMAGE", "unused") + .env("APPROXBENCH", "unused") + .output() + .unwrap(); + assert!(!out.status.success()); + assert!(String::from_utf8_lossy(&out.stderr).contains("was not made by this script")); + assert!(dir.join("keep").exists()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/aqpbm-engines-external/tests/fixtures/plans/REV b/aqpbm-engines-external/tests/fixtures/plans/REV index 0d579c20..283d480c 100644 --- a/aqpbm-engines-external/tests/fixtures/plans/REV +++ b/aqpbm-engines-external/tests/fixtures/plans/REV @@ -18,6 +18,11 @@ Named fixtures (pretty-printed): joint_5m five 5m queries over latency and requests_total capabilities/clickhouse.equivalent-only.json (workloads/grid/joint__5m__1hz__60s.json) + probe_sum_10s sum_over_time(probe[10s]), every 10 s (the harness conformance probe) capabilities/clickhouse.equivalent-only.json + (workloads/probe/probe_sum_10s.json) + probe_count_10s count_over_time(probe[10s]), every 10 s capabilities/clickhouse.equivalent-only.json + (workloads/probe/probe_count_10s.json) + Public plans (`public/..plan.json`, compact): the workload `workloads//.json` planned with `capabilities/.json`. They are the eight RUN-1 showcase shapes (1 m windows, 1 Hz, every 10 s), `mixed-03`, diff --git a/aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json b/aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json new file mode 100644 index 00000000..a19b9e97 --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/plans/probe_count_10s.plan.json @@ -0,0 +1,307 @@ +{ + "wire_version": 15, + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "NonASAP": { + "Scan": { + "source": { + "TimeSeries": { + "metric": "probe" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": "None", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "retention_ms": 10000 + }, + { + "id": 1, + "payload": { + "NonASAP": { + "TimeRange": { + "range": { + "secs": 10, + "nanos": 0 + }, + "kind": "range", + "child": 0 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + }, + { + "id": 2, + "payload": { + "NonASAP": { + "Aggregate": { + "reduction": "PerEntity", + "measures": [ + { + "kind": "count" + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null, + "child": 1 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + } + ], + "roots": [ + 2 + ], + "scalar_roots": [] + }, + "query_texts": [ + "count_over_time(probe[10s])" + ] +} \ No newline at end of file diff --git a/aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json b/aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json new file mode 100644 index 00000000..b7574ea2 --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/plans/probe_sum_10s.plan.json @@ -0,0 +1,308 @@ +{ + "wire_version": 15, + "dag": { + "nodes": [ + { + "id": 0, + "payload": { + "NonASAP": { + "Scan": { + "source": { + "TimeSeries": { + "metric": "probe" + } + }, + "predicates": [], + "schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": "None", + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "retention_ms": 10000 + }, + { + "id": 1, + "payload": { + "NonASAP": { + "TimeRange": { + "range": { + "secs": 10, + "nanos": 0 + }, + "kind": "range", + "child": 0 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + }, + { + "id": 2, + "payload": { + "NonASAP": { + "Aggregate": { + "reduction": "PerEntity", + "measures": [ + { + "kind": "sum", + "col": null + } + ], + "output_names": [ + "" + ], + "filters": [], + "having": null, + "child": 1 + } + } + }, + "output_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "lifetime": { + "OneEvaluation": "Memory" + }, + "output_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + } + } + ], + "edges": [ + { + "producer": 0, + "consumer": 1, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + }, + { + "producer": 1, + "consumer": 2, + "role": "Input", + "intermediate_schema": { + "fields": [ + { + "name": "ts", + "dtype": { + "Plain": "timestamp" + }, + "nullable": false + }, + { + "name": "value", + "dtype": { + "Plain": "float64" + }, + "nullable": false + }, + { + "name": "$promql_series_identity", + "dtype": { + "Plain": "utf8" + }, + "nullable": false + } + ], + "time_index": 0, + "unique_keys": [ + [ + 2, + 0 + ] + ], + "closed": true + }, + "data_state": { + "timing": "query_time", + "primitive": "Raw" + }, + "grouping": "NotApplicable", + "window": "NotApplicable" + } + ], + "roots": [ + 2 + ], + "scalar_roots": [] + }, + "query_texts": [ + "sum_over_time(probe[10s])" + ] +} \ No newline at end of file diff --git a/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json new file mode 100644 index 00000000..d3b3eee6 --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_count_10s.json @@ -0,0 +1,36 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "count_over_time(probe[10s])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": {}, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 0.1 + } + }, + "sources": { + "probe": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 10000 + } + } + } +} \ No newline at end of file diff --git a/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json new file mode 100644 index 00000000..cfda417c --- /dev/null +++ b/aqpbm-engines-external/tests/fixtures/workloads/probe/probe_sum_10s.json @@ -0,0 +1,36 @@ +{ + "query_workload": { + "language": "promql", + "repeating_queries": [ + { + "query": "sum_over_time(probe[10s])", + "demand": { + "fixed_interval_at": { + "interval": 10000, + "evaluation_phase": 0 + } + }, + "requirements": {}, + "predictability": { + "predictable": { + "known_at": null + } + } + } + ] + }, + "data_workload": { + "arrival": { + "ingesting": { + "rows_per_second": 0.1 + } + }, + "sources": { + "probe": { + "series_count": 1, + "metric_type": "gauge", + "sample_interval": 10000 + } + } + } +} \ No newline at end of file diff --git a/scripts/extbench-up.sh b/scripts/extbench-up.sh new file mode 100755 index 00000000..c979482f --- /dev/null +++ b/scripts/extbench-up.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Start (or stop) one measured server for `approxbench bench-external` on +# this host: a docker container pinned to given cores, with the cgroup +# agent beside it. +# +# scripts/extbench-up.sh clickhouse up [] +# scripts/extbench-up.sh clickhouse down [] +# scripts/extbench-up.sh check-listen-host
+# +# Example: LISTEN_HOST=10.0.0.2 scripts/extbench-up.sh clickhouse up /data/run-1/clickhouse 2,22 0 18123 19101 +# +# Environment: +# LISTEN_HOST address the server and the agent listen on (required): +# a private (10/8, 172.16/12, 192.168/16) or loopback +# address, never 0.0.0.0 or a public one: the server +# has no password +# CLICKHOUSE_BINARY path of the ClickHouse binary to bind-mount (required) +# CLICKHOUSE_SHA256 its expected sha256 (required); a mismatch refuses +# BASE_IMAGE the image the binary runs in, pinned by digest (required) +# APPROXBENCH path of the approxbench binary, for `cgroup-agent` (required) +# AGENT_CORES cores the agent may run on (default: every core); keep +# them off the server's cores +# +# The agent runs outside the container, so its reads are never charged to +# the server's group. +# +# Every start begins from an empty data dir. The script only deletes a data +# dir it created (it holds the marker file .extbench-data-dir). The +# container runs as the invoking user, with no CPU quota (--cpus unset) and +# no memory limit. ClickHouse runs on its embedded configuration: without a +# main config file it reads no config.d, so the few settings changed here +# are `--key=value` overrides. The embedded configuration also defines the +# native, MySQL, PostgreSQL and interserver ports (a port of 0 binds a random +# one), so they get the four ports after the HTTP port, on LISTEN_HOST too. +set -euo pipefail + +marker=.extbench-data-dir + +# Exit 0 when $1 is a private or loopback IPv4 address. +private_address() { + local a b c d + IFS=. read -r a b c d <<<"$1" || return 1 + for octet in "$a" "$b" "$c" "$d"; do + [[ "$octet" =~ ^[0-9]+$ ]] && [ "$octet" -le 255 ] || return 1 + done + [ "$a" = 10 ] && return 0 + [ "$a" = 127 ] && return 0 + [ "$a" = 172 ] && [ "$b" -ge 16 ] && [ "$b" -le 31 ] && return 0 + [ "$a" = 192 ] && [ "$b" = 168 ] && return 0 + return 1 +} + +if [ "${1:-}" = check-listen-host ]; then + private_address "${2:?address}" || { echo "refused: ${2} is not a private address" >&2; exit 1; } + exit 0 +fi + +engine=${1:?engine}; action=${2:?up|down}; data_dir=$(realpath -m "${3:?data dir}") + +# Kill the agent recorded in $data_dir/agent.pid, if that pid is still it. +stop_agent() { + local pid + [ -f "$data_dir/agent.pid" ] || return 0 + pid=$(cat "$data_dir/agent.pid") + if [ -r "/proc/$pid/cmdline" ] && tr '\0' ' ' < "/proc/$pid/cmdline" | grep -q cgroup-agent; then + kill "$pid" 2>/dev/null || true + fi +} +[ "$engine" = clickhouse ] || { echo "unknown engine $engine" >&2; exit 2; } + +if [ "$action" = down ]; then + name=${4:-extbench-clickhouse} + stop_agent + docker rm -f "$name" >/dev/null 2>&1 || true + exit 0 +fi + +cores=${4:?cores}; numa=${5:?numa node}; http_port=${6:?http port}; agent_port=${7:?agent port} +name=${8:-extbench-clickhouse} +: "${LISTEN_HOST:?set LISTEN_HOST to a private address}" "${CLICKHOUSE_BINARY:?}" \ + "${CLICKHOUSE_SHA256:?}" "${BASE_IMAGE:?}" "${APPROXBENCH:?}" +private_address "$LISTEN_HOST" || { echo "refused: LISTEN_HOST $LISTEN_HOST is not a private address" >&2; exit 1; } +CLICKHOUSE_BINARY=$(realpath "$CLICKHOUSE_BINARY") +actual=$(sha256sum "$CLICKHOUSE_BINARY" | cut -d' ' -f1) +[ "$actual" = "$CLICKHOUSE_SHA256" ] || { echo "binary sha256 $actual != $CLICKHOUSE_SHA256" >&2; exit 1; } + +if [ -e "$data_dir" ]; then + [ -f "$data_dir/$marker" ] || { echo "refused: $data_dir exists and was not made by this script" >&2; exit 1; } + stop_agent + docker rm -f "$name" >/dev/null 2>&1 || true + rm -rf "$data_dir" +fi +mkdir -p "$data_dir/db"; touch "$data_dir/$marker" + +docker rm -f "$name" >/dev/null 2>&1 || true +docker run -d --name "$name" --network host \ + --cpuset-cpus "$cores" --cpuset-mems "$numa" --user "$(id -u):$(id -g)" \ + -v "$CLICKHOUSE_BINARY:/usr/bin/clickhouse:ro" -v "$data_dir/db:/var/lib/clickhouse" \ + -w /var/lib/clickhouse \ + "$BASE_IMAGE" /usr/bin/clickhouse server -- --path=/var/lib/clickhouse/ \ + --http_port="$http_port" --tcp_port=$((http_port + 1)) --mysql_port=$((http_port + 2)) \ + --postgresql_port=$((http_port + 3)) --interserver_http_port=$((http_port + 4)) \ + --listen_host="$LISTEN_HOST" --logger.level=warning --display_name="$name" >/dev/null +for _ in $(seq 1 60); do + curl -sf "http://$LISTEN_HOST:$http_port/ping" >/dev/null && break; sleep 1 +done +curl -sf "http://$LISTEN_HOST:$http_port/ping" >/dev/null || { echo "the server did not start" >&2; docker logs "$name" >&2; exit 1; } + +id=$(docker inspect -f '{{.Id}}' "$name") +group="/sys/fs/cgroup/system.slice/docker-$id.scope" +[ -d "$group" ] || { echo "no cgroup at $group (is docker on the systemd cgroup driver?)" >&2; exit 1; } +pin=() +[ -n "${AGENT_CORES:-}" ] && pin=(taskset -c "$AGENT_CORES") +nohup "${pin[@]}" "$APPROXBENCH" cgroup-agent --cgroup-dir "$group" --listen "$LISTEN_HOST:$agent_port" \ + > "$data_dir/agent.log" 2>&1 & echo $! > "$data_dir/agent.pid" +for _ in $(seq 1 20); do + curl -sf "http://$LISTEN_HOST:$agent_port/cpu.stat" >/dev/null && break; sleep 0.5 +done +curl -sf "http://$LISTEN_HOST:$agent_port/cpu.stat" >/dev/null \ + || { echo "the cgroup agent did not start:" >&2; cat "$data_dir/agent.log" >&2; exit 1; } + +query() { curl -sf "http://$LISTEN_HOST:$http_port/" --data-binary "$1"; } +siblings() { + for core in ${cores//,/ }; do + printf '%s:%s ' "$core" "$(cat "/sys/devices/system/cpu/cpu$core/topology/thread_siblings_list")" + done +} +{ echo "binary_sha256 $actual" + echo "version $(query "SELECT version()")" + echo "git_hash $(query "SELECT value FROM system.build_options WHERE name = 'VERSION_GITHASH'")" + echo "base_image $BASE_IMAGE" + echo "cpuset $cores mems $numa" + echo "thread_siblings $(siblings)" + echo "cpu_model $(lscpu | sed -n 's/^Model name: *//p')" + echo "cgroup $group" + echo "agent_cores ${AGENT_CORES:-all}" + echo "kernel $(uname -r)"; } > "$data_dir/versions.txt" +cat "$data_dir/versions.txt"