From 353ebc5f205cd864d2cc78e9fc099f6dbf85c619 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:23:06 +0000 Subject: [PATCH 1/3] test(planner): MetricsQL per-series queries plan, install and answer as PromQL Four queries over 1,000 series at 1 Hz, every 10 s within relative error 0.01, written in both languages: each MetricsQL query plans and installs, selects the PromQL form's DAG, and answers what the PromQL form answers on synthetic samples (and the hand-worked answers). They fail on main: MetricsQL roots never get the series identity. The answer-row helper moves to the shared time_series test module. --- crates/planner/tests/main.rs | 1 + crates/planner/tests/metricsql_workloads.rs | 233 ++++++++++++++++++++ crates/planner/tests/runtime_install.rs | 26 +-- crates/planner/tests/time_series/mod.rs | 22 ++ 4 files changed, 258 insertions(+), 24 deletions(-) create mode 100644 crates/planner/tests/metricsql_workloads.rs diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index d16b381fa..d72a470e7 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -7,6 +7,7 @@ mod time_series; mod columnar_batches; mod e2e_plan; mod exact_only_reads; +mod metricsql_workloads; mod reduce_aggregates; mod runtime_install; mod sql_table_scans; diff --git a/crates/planner/tests/metricsql_workloads.rs b/crates/planner/tests/metricsql_workloads.rs new file mode 100644 index 000000000..82ef37bda --- /dev/null +++ b/crates/planner/tests/metricsql_workloads.rs @@ -0,0 +1,233 @@ +//! A MetricsQL workload plans, installs and answers as its PromQL form +//! does: both languages read time series, so each query root carries every +//! series' full identity as a column before Stage 0, whichever of the two +//! frontends lowered it. +//! +//! The workload: 1,000 series per metric sampled every second, each query +//! every 10 s within relative error 0.01. Without the series identity a +//! MetricsQL `rate`, `increase` or `max_over_time` root has no valid plan +//! ("per-entity summary requires complete source identity") and `sum` over +//! an instant selector selects a plan whose output type differs from the +//! query's. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use asap_executor::physical_planner::compile; +use asap_executor::runtime::{Limits, RunContext, Scope}; +use asap_executor::values::Batch; +use asap_planner::{ + plan_installable, FrontendInput, InstallablePlan, PlanningModels, PlanningWorkload, UserInput, +}; +use futures::{executor::block_on, StreamExt}; + +use crate::time_series::{answers, time_series_tables, Answer, RawDataSample}; + +/// When every query is evaluated: the end of the first hour. +const EVALUATION_MS: i64 = 3_600_000; + +/// Queries both languages write the same way, each over one of the two +/// metrics [`workload`] declares. +const QUERIES: [&str; 4] = [ + "sum by (job)(rate(requests_total[5m]))", + "increase(requests_total[5m])", + "max_over_time(latency[5m])", + r#"sum(latency{job="j1"})"#, +]; + +/// `query` written in `language` ("promql" or "metricsql"), every 10 s +/// within relative error 0.01, over 1,000 `requests_total` counter series +/// and 1,000 `latency` gauge series, each sampled every second. +fn workload(language: &str, query: &str) -> PlanningWorkload { + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "{language}", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.01}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 2000.0}}}}, + "sources": {{ + "requests_total": {{"series_count": 1000, "metric_type": "counter", + "sample_interval": 1000}}, + "latency": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 1000}}}}}}}}"# + )) + .unwrap() +} + +/// The built-in plan of `query` written in MetricsQL, or the error the +/// Planner returned. +async fn plan_metricsql(query: &str) -> Result { + let workload = workload("metricsql", query); + plan_installable(UserInput::new( + &workload, + FrontendInput::Metricsql, + PlanningModels::builtin(), + )) + .await + .map(|(plan, _report)| plan) + .map_err(|e| e.to_string()) +} + +/// The built-in plan of `query` written in PromQL. +async fn plan_promql(query: &str) -> InstallablePlan { + let workload = workload("promql", query); + plan_installable(UserInput::new( + &workload, + FrontendInput::Promql, + PlanningModels::builtin(), + )) + .await + .unwrap_or_else(|e| panic!("{query} as PromQL: {e}")) + .0 +} + +/// One sample of `metric` for series `(instance, job)`. +fn sample(metric: &str, instance: &str, job: &str, timestamp_ms: i64, value: f64) -> RawDataSample { + RawDataSample { + labels: BTreeMap::from([ + ("__name__".to_string(), metric.to_string()), + ("instance".to_string(), instance.to_string()), + ("job".to_string(), job.to_string()), + ]), + timestamp_ms, + value, + } +} + +/// Four series of each metric (instances `a`..`d`, jobs `j1` and `j2`), +/// sampled every 15 s from 8 minutes before [`EVALUATION_MS`] through it: +/// `requests_total` grows by `index + 1` per sample, and `latency` cycles +/// through `index * 10 + step % 7`. +fn samples() -> Vec { + let series = [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")]; + let mut samples = Vec::new(); + for (index, (instance, job)) in series.iter().enumerate() { + for step in 0..=32i64 { + let timestamp_ms = EVALUATION_MS - 480_000 + step * 15_000; + let count = (index as f64 + 1.0) * step as f64; + let latency = index as f64 * 10.0 + (step % 7) as f64; + samples.push(sample("requests_total", instance, job, timestamp_ms, count)); + samples.push(sample("latency", instance, job, timestamp_ms, latency)); + } + } + samples +} + +/// The answers of `plan`'s one query at [`EVALUATION_MS`] over `samples`, +/// sorted by labels: the executor compiles the plan and runs its root at +/// query time over the raw samples. +fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec { + let roots: Vec = plan.dag.roots.iter().map(|&root| root as u64).collect(); + let compiled = compile(&plan.dag, &[], &roots).expect("the executor compiles the plan"); + let instance = compiled + .instantiate(Arc::new(time_series_tables(&plan.dag, samples))) + .unwrap(); + let batches: Vec = block_on(async { + let scope = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + let context = RunContext::new(scope, Limits::default()).unwrap(); + let mut output = instance + .execute(compiled.roots(), context) + .unwrap() + .remove(0); + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch.unwrap()).clone()); + } + batches + }); + answers(&batches) +} + +/// Each query written in MetricsQL plans and installs: the executor +/// compiles its plan. +#[tokio::test] +async fn metricsql_queries_plan_and_install() { + let mut failures = Vec::new(); + for query in QUERIES { + let installed = plan_metricsql(query) + .await + .and_then(|plan| plan.check_compiles()); + if let Err(error) = installed { + failures.push(format!("{query}: {error}")); + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// Each query's MetricsQL plan is its PromQL plan: the same selected DAG, +/// node for node. +#[tokio::test] +async fn metricsql_queries_select_the_promql_plan() { + for query in QUERIES { + let metricsql = plan_metricsql(query) + .await + .unwrap_or_else(|e| panic!("{query} as MetricsQL: {e}")); + let promql = plan_promql(query).await; + assert_eq!(metricsql.dag, promql.dag, "{query}"); + } +} + +/// Each query's MetricsQL plan answers what its PromQL plan answers on the +/// same samples, and both match the answers worked out by hand from +/// [`samples`] at T. `increase` over 5 m sees 20 samples per series, +/// `(index + 1) * 19` apart over 285 s, extrapolated to the 300 s window: +/// `(index + 1) * 20`. `max_over_time` over 5 m sees steps 13..=32, whose +/// largest `step % 7` is 6. `sum(latency{job="j1"})` reads each `j1` +/// series' sample at T (step 32, `32 % 7` = 4): 4 + 14 = 18. The grouped +/// counter query answers no rows in either language today (issue #1338), +/// so it checks only that the two languages agree. +#[tokio::test] +async fn metricsql_queries_answer_as_promql_does() { + let series = |instance: &str, job: &str| { + BTreeMap::from([ + ("instance".to_string(), instance.to_string()), + ("job".to_string(), job.to_string()), + ]) + }; + let per_series = |values: [f64; 4]| -> Option> { + let names = [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")]; + Some( + names + .iter() + .zip(values) + .map(|(&(i, j), v)| (series(i, j), v)) + .collect(), + ) + }; + // Each query's hand-worked answers; `None` where only the languages' + // agreement is checked. + let expected: [Option>; 4] = [ + None, + per_series([20.0, 40.0, 60.0, 80.0]), + per_series([6.0, 16.0, 26.0, 36.0]), + Some(vec![(BTreeMap::new(), 18.0)]), + ]; + let samples = samples(); + for (query, expected) in QUERIES.into_iter().zip(expected) { + let metricsql = plan_metricsql(query) + .await + .unwrap_or_else(|e| panic!("{query} as MetricsQL: {e}")); + let metricsql_answers = query_answers(&metricsql, &samples); + assert_eq!( + metricsql_answers, + query_answers(&plan_promql(query).await, &samples), + "{query}" + ); + let Some(expected) = expected else { continue }; + assert_eq!( + metricsql_answers.len(), + expected.len(), + "{query}: {metricsql_answers:?}" + ); + for (answer, expected) in metricsql_answers.iter().zip(&expected) { + assert_eq!(answer.0, expected.0, "{query}: {metricsql_answers:?}"); + assert!( + (answer.1 - expected.1).abs() <= 1e-9 * expected.1, + "{query}: {metricsql_answers:?}" + ); + } + } +} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index d4d85ebe8..396ae7d92 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -12,7 +12,7 @@ use asap_executor::runtime::{OutputStream, RunContext, Scope}; use asap_executor::sources::{ DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, }; -use asap_executor::values::{Batch, Value}; +use asap_executor::values::Batch; use asap_executor::Error as ExecutorError; use asap_planner::{ plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, @@ -22,7 +22,7 @@ use asap_planner::{ use asap_types::ir::operator::Source; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use time_series::{run, sample, series_labels, time_series_tables, RawDataSample}; +use time_series::{answers, run, sample, time_series_tables, Answer, RawDataSample}; /// Example 4b's evaluation time: the end of the 60-minute window. const EVALUATION_MS: i64 = 3_600_000; @@ -89,28 +89,6 @@ impl DataStore for CountingStore { } } -/// One answer row: its series labels and its value. -type Answer = (BTreeMap, f64); - -/// Each answer row as its series labels and value, sorted by labels. -fn answers(batches: &[Batch]) -> Vec { - let mut answers: Vec<_> = batches - .iter() - .flat_map(|batch| { - batch.to_rows().into_iter().map(|row| { - let labels = series_labels(batch.schema(), &row); - let value = row.iter().find_map(|v| match v { - Value::Float64(v) => Some(*v), - _ => None, - }); - (labels, value.expect("a Float64 answer")) - }) - }) - .collect(); - answers.sort_by(|a, b| a.0.cmp(&b.0)); - answers -} - /// The plan for `queries` on a deployment that keeps no raw samples, and /// Stage 3's report of the selection it made (which the plan file does not /// carry). diff --git a/crates/planner/tests/time_series/mod.rs b/crates/planner/tests/time_series/mod.rs index 49efb8381..d1891366d 100644 --- a/crates/planner/tests/time_series/mod.rs +++ b/crates/planner/tests/time_series/mod.rs @@ -166,3 +166,25 @@ pub fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap } labels } + +/// One answer row: its series labels and its value. +pub type Answer = (BTreeMap, f64); + +/// Each answer row as its series labels and value, sorted by labels. +pub fn answers(batches: &[Batch]) -> Vec { + let mut answers: Vec<_> = batches + .iter() + .flat_map(|batch| { + batch.to_rows().into_iter().map(|row| { + let labels = series_labels(batch.schema(), &row); + let value = row.iter().find_map(|v| match v { + Value::Float64(v) => Some(*v), + _ => None, + }); + (labels, value.expect("a Float64 answer")) + }) + }) + .collect(); + answers.sort_by(|a, b| a.0.cmp(&b.0)); + answers +} From 1d92c8bf59d7a0cae91be2bfbafe489ecb24a2a1 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:23:09 +0000 Subject: [PATCH 2/3] fix(planner): give MetricsQL roots the series identity Stage1ToStage3Pass added the series-identity column only to PromQL roots, so MetricsQL per-series queries had no installable plan. One rule now decides which languages carry it: QueryLanguage::has_series_identity (PromQL and MetricsQL). Closes #1337 --- crates/planner/src/run.rs | 21 +++++++++++---------- crates/types/src/workload/query.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 1f69236bc..c2aa62cf5 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -17,7 +17,7 @@ use asap_types::ir::schema::series_identity::{with_promql_series_identity, Serie use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::stages::{LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates}; -use asap_types::workload::{DataWorkload, QueryLanguage, QueryWorkloadEntry}; +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::models::PlanningModels; use crate::pass::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput}; @@ -169,12 +169,13 @@ fn elapsed_ms(started: Instant) -> f64 { } /// The planner pipeline behind the optimization-pass interface: the -/// workload's roots in entry order, PromQL roots with each series' full -/// identity as a column (the row representation per-series state needs at -/// runtime; a root with no such representation is planned over its labels -/// alone), through [`run_planning`] with every entry's accuracy target. The -/// output is the plan Stage 3 built; its `Selection` report is -/// [`run_planning`]'s. +/// workload's roots in entry order, those of a time-series language +/// ([`asap_types::workload::QueryLanguage::has_series_identity`]: PromQL and +/// MetricsQL) with each series' full identity as a column (the row +/// representation per-series state needs at runtime; a root with no such +/// representation is planned over its labels alone), through +/// [`run_planning`] with every entry's accuracy target. The output is the +/// plan Stage 3 built; its `Selection` report is [`run_planning`]'s. #[derive(Debug, Default, Clone, Copy)] pub struct Stage1ToStage3Pass; @@ -185,13 +186,13 @@ impl OptimizationPass for Stage1ToStage3Pass { fn optimize(&self, input: OptimizationInput<'_>) -> Result { let workload = input.workload; - let promql = workload.query_workload().language == QueryLanguage::PromQL; + let series_identity = workload.query_workload().language.has_series_identity(); let mut roots: Vec<(usize, QueryRoot)> = workload .operator_indices() .iter() .zip(workload.exprs()) .map(|(&index, node)| { - let node = match promql { + let node = match series_identity { true => promql_root(node, with_promql_series_identity(node))?, false => Rc::clone(node), }; @@ -221,7 +222,7 @@ impl OptimizationPass for Stage1ToStage3Pass { } } -/// The PromQL root `node` planned: `realized`, its series identity +/// The PromQL or MetricsQL root `node` planned: `realized`, its series identity /// realization (`with_promql_series_identity`), or `node` itself, over its /// label columns, when the realization does not apply to one of its /// operators (`limitk(2, m)`'s `PromqlSeriesSample`, `info(m)`'s diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index 2e5c64d2f..55692ec4f 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -57,6 +57,19 @@ pub enum QueryLanguage { ElasticDSL, } +impl QueryLanguage { + /// Whether this language's queries read time series, whose rows the + /// planner keys by each series' full label set: the + /// [`PROMQL_SERIES_IDENTITY`](crate::ir::schema::PROMQL_SERIES_IDENTITY) + /// column, added to every root before Stage 0. PromQL and MetricsQL + /// do: `rate(requests_total[5m])` keeps one rate per series, each + /// series `{"__name__":"requests_total","instance":"a","job":"j1"}` + /// as one identity. SQL reads tables, whose rows have no such identity. + pub fn has_series_identity(&self) -> bool { + matches!(self, Self::PromQL | Self::MetricsQL) + } +} + /// Independent accuracy and response-latency constraints attached to one /// query in the workload. /// @@ -889,6 +902,22 @@ mod tests { } } + /// The two time-series languages carry the series identity; the table + /// languages do not. + #[test] + fn only_time_series_languages_have_series_identity() { + let cases = [ + (QueryLanguage::PromQL, true), + (QueryLanguage::MetricsQL, true), + (QueryLanguage::SQL(SqlDialect::DataFusionSQL), false), + (QueryLanguage::DataFusion, false), + (QueryLanguage::ElasticDSL, false), + ]; + for (language, expected) in cases { + assert_eq!(language.has_series_identity(), expected, "{language:?}"); + } + } + #[test] fn struct_enum_variants_reject_unknown_fields() { assert!( From 562e50b000cca7a2f62d262cc08399b8dafcdf5c Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:56:17 +0000 Subject: [PATCH 3/3] refactor(planner): share the run and answer helpers; rename to reads_time_series - QueryLanguage::has_series_identity becomes reads_time_series: the language reads time series (PromQL, MetricsQL) or tables (SQL, DataFusion, ElasticDSL). The identity-column explanation moves to its one call site in Stage1ToStage3Pass::optimize. - The planner tests' time_series module now holds sample, query_roots, run(dag, store, ids, scope) and assert_answers_near (absolute) / assert_answers_near_relative; runtime_install and metricsql_workloads use them and drop their copies. The write-only read counter (CountingStore, RAW_DATA_READS) that no test read is deleted. - New test: MetricsQL avg by (job)(latency) answers j1 9, j2 29 as PromQL does. Without the identity it planned and ran without error but answered 14 and 34. --- crates/planner/src/run.rs | 9 +- crates/planner/tests/metricsql_workloads.rs | 91 +++++++--------- crates/planner/tests/runtime_install.rs | 84 +++------------ crates/planner/tests/time_series/mod.rs | 109 ++++++++++++++------ crates/types/src/workload/query.rs | 22 ++-- 5 files changed, 149 insertions(+), 166 deletions(-) diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index c2aa62cf5..2caf7e752 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -170,7 +170,7 @@ fn elapsed_ms(started: Instant) -> f64 { /// The planner pipeline behind the optimization-pass interface: the /// workload's roots in entry order, those of a time-series language -/// ([`asap_types::workload::QueryLanguage::has_series_identity`]: PromQL and +/// ([`asap_types::workload::QueryLanguage::reads_time_series`]: PromQL and /// MetricsQL) with each series' full identity as a column (the row /// representation per-series state needs at runtime; a root with no such /// representation is planned over its labels alone), through @@ -186,7 +186,12 @@ impl OptimizationPass for Stage1ToStage3Pass { fn optimize(&self, input: OptimizationInput<'_>) -> Result { let workload = input.workload; - let series_identity = workload.query_workload().language.has_series_identity(); + // Time-series rows are keyed by each series' full label set: the + // `PROMQL_SERIES_IDENTITY` column, added to every root before Stage + // 0. `rate(requests_total[5m])` keeps one rate per series, each + // series `{"__name__":"requests_total","instance":"a","job":"j1"}` + // one identity. Table rows have no such identity. + let series_identity = workload.query_workload().language.reads_time_series(); let mut roots: Vec<(usize, QueryRoot)> = workload .operator_indices() .iter() diff --git a/crates/planner/tests/metricsql_workloads.rs b/crates/planner/tests/metricsql_workloads.rs index 82ef37bda..c95de25e5 100644 --- a/crates/planner/tests/metricsql_workloads.rs +++ b/crates/planner/tests/metricsql_workloads.rs @@ -11,17 +11,16 @@ //! query's. use std::collections::BTreeMap; -use std::sync::Arc; -use asap_executor::physical_planner::compile; -use asap_executor::runtime::{Limits, RunContext, Scope}; -use asap_executor::values::Batch; +use asap_executor::runtime::Scope; use asap_planner::{ plan_installable, FrontendInput, InstallablePlan, PlanningModels, PlanningWorkload, UserInput, }; -use futures::{executor::block_on, StreamExt}; -use crate::time_series::{answers, time_series_tables, Answer, RawDataSample}; +use crate::time_series::{ + answers, assert_answers_near_relative, query_roots, run, sample, time_series_tables, Answer, + RawDataSample, +}; /// When every query is evaluated: the end of the first hour. const EVALUATION_MS: i64 = 3_600_000; @@ -82,19 +81,6 @@ async fn plan_promql(query: &str) -> InstallablePlan { .0 } -/// One sample of `metric` for series `(instance, job)`. -fn sample(metric: &str, instance: &str, job: &str, timestamp_ms: i64, value: f64) -> RawDataSample { - RawDataSample { - labels: BTreeMap::from([ - ("__name__".to_string(), metric.to_string()), - ("instance".to_string(), instance.to_string()), - ("job".to_string(), job.to_string()), - ]), - timestamp_ms, - value, - } -} - /// Four series of each metric (instances `a`..`d`, jobs `j1` and `j2`), /// sampled every 15 s from 8 minutes before [`EVALUATION_MS`] through it: /// `requests_total` grows by `index + 1` per sample, and `latency` cycles @@ -107,8 +93,9 @@ fn samples() -> Vec { let timestamp_ms = EVALUATION_MS - 480_000 + step * 15_000; let count = (index as f64 + 1.0) * step as f64; let latency = index as f64 * 10.0 + (step % 7) as f64; - samples.push(sample("requests_total", instance, job, timestamp_ms, count)); - samples.push(sample("latency", instance, job, timestamp_ms, latency)); + let labels = [("instance", *instance), ("job", *job)]; + samples.push(sample("requests_total", &labels, timestamp_ms, count)); + samples.push(sample("latency", &labels, timestamp_ms, latency)); } } samples @@ -118,27 +105,11 @@ fn samples() -> Vec { /// sorted by labels: the executor compiles the plan and runs its root at /// query time over the raw samples. fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec { - let roots: Vec = plan.dag.roots.iter().map(|&root| root as u64).collect(); - let compiled = compile(&plan.dag, &[], &roots).expect("the executor compiles the plan"); - let instance = compiled - .instantiate(Arc::new(time_series_tables(&plan.dag, samples))) - .unwrap(); - let batches: Vec = block_on(async { - let scope = Scope::Query { - evaluation_time_ms: EVALUATION_MS, - }; - let context = RunContext::new(scope, Limits::default()).unwrap(); - let mut output = instance - .execute(compiled.roots(), context) - .unwrap() - .remove(0); - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.unwrap()).clone()); - } - batches - }); - answers(&batches) + let scope = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + let store = time_series_tables(&plan.dag, samples); + answers(&run(&plan.dag, store, &query_roots(plan), scope)[0]) } /// Each query written in MetricsQL plans and installs: the executor @@ -217,17 +188,29 @@ async fn metricsql_queries_answer_as_promql_does() { "{query}" ); let Some(expected) = expected else { continue }; - assert_eq!( - metricsql_answers.len(), - expected.len(), - "{query}: {metricsql_answers:?}" - ); - for (answer, expected) in metricsql_answers.iter().zip(&expected) { - assert_eq!(answer.0, expected.0, "{query}: {metricsql_answers:?}"); - assert!( - (answer.1 - expected.1).abs() <= 1e-9 * expected.1, - "{query}: {metricsql_answers:?}" - ); - } + assert_answers_near_relative(&metricsql_answers, &expected, 1e-9); } } + +/// `avg by (job)(latency)` in MetricsQL averages each job's series at T, as +/// its PromQL form does: `j1` holds series `a` (latency 4 at step 32) and +/// `b` (14), so 9; `j2` holds `c` (24) and `d` (34), so 29. A plan whose +/// rows lack the series identity plans and runs without any error, yet +/// answers a different average, so this checks the values, not only that +/// the query plans. +#[tokio::test] +async fn metricsql_grouped_average_averages_every_series_of_each_group() { + let query = "avg by (job)(latency)"; + let job = |name: &str| BTreeMap::from([("job".to_string(), name.to_string())]); + let expected = vec![(job("j1"), 9.0), (job("j2"), 29.0)]; + let samples = samples(); + let metricsql = plan_metricsql(query) + .await + .unwrap_or_else(|e| panic!("{query} as MetricsQL: {e}")); + let metricsql_answers = query_answers(&metricsql, &samples); + assert_answers_near_relative(&metricsql_answers, &expected, 1e-9); + assert_eq!( + metricsql_answers, + query_answers(&plan_promql(query).await, &samples) + ); +} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index 396ae7d92..6f1d6348a 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -4,25 +4,20 @@ //! the compiler checks that no internal is used (CI runs //! `cargo check -p asap-planner --tests --locked` on its own for that). -use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet}; use crate::time_series; -use asap_executor::runtime::{OutputStream, RunContext, Scope}; -use asap_executor::sources::{ - DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, -}; -use asap_executor::values::Batch; -use asap_executor::Error as ExecutorError; +use asap_executor::runtime::Scope; use asap_planner::{ plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, - MaterializedView, OperatorKind, PlanningModels, PlanningWorkload, SelectedPlanReport, - UserInput, + OperatorKind, PlanningModels, PlanningWorkload, SelectedPlanReport, UserInput, }; -use asap_types::ir::operator::Source; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use time_series::{answers, run, sample, time_series_tables, Answer, RawDataSample}; +use time_series::{ + answers, assert_answers_near, query_roots, run, sample, time_series_tables, Answer, + RawDataSample, +}; /// Example 4b's evaluation time: the end of the 60-minute window. const EVALUATION_MS: i64 = 3_600_000; @@ -60,35 +55,6 @@ fn workload_every(queries: &[(&str, u64)]) -> PlanningWorkload { .unwrap() } -thread_local! { - /// How many `read_raw_data` calls the runs on this thread made. - static RAW_DATA_READS: Cell = const { Cell::new(0) }; -} - -/// A store counting its `read_raw_data` calls in [`RAW_DATA_READS`]. -struct CountingStore(InMemoryDataStore); - -impl DataStore for CountingStore { - fn read_raw_data( - &self, - source: &Source, - request: &RawDataRequest, - context: RunContext, - ) -> Result, ExecutorError> { - RAW_DATA_READS.with(|reads| reads.set(reads.get() + 1)); - self.0.read_raw_data(source, request, context) - } - - fn read_materialized_view( - &self, - view: &MaterializedView, - request: &MaterializedViewRequest, - context: RunContext, - ) -> Result, ExecutorError> { - self.0.read_materialized_view(view, request, context) - } -} - /// The plan for `queries` on a deployment that keeps no raw samples, and /// Stage 3's report of the selection it made (which the plan file does not /// carry). @@ -126,11 +92,6 @@ async fn plan_with_report_for( .unwrap() } -/// The node that answers each query of `plan`, in workload order. -fn query_roots(plan: &InstallablePlan) -> Vec { - plan.dag.roots.iter().map(|&root| root as u64).collect() -} - /// The plan file carries Stage 3's predicted usage of the selected plan, so /// a runtime reports it against what it measures without planning again: /// for Example 4b's tumbling windows maintained at ingestion time, its nodes' costs @@ -274,7 +235,7 @@ async fn an_instant_selector_reads_the_latest_sample_per_series() { let query = Scope::Query { evaluation_time_ms: EVALUATION_MS, }; - let store = CountingStore(time_series_tables(dag, &samples)); + let store = time_series_tables(dag, &samples); let answered = answers(&run(dag, store, &query_roots(&plan), query)[0]); assert_eq!(answered.len(), 1); assert_eq!(answered[0].1, 110.0); @@ -490,28 +451,15 @@ fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec Vec> { - // The executor's plan for `ids`; every `Scan` it reaches is a raw data - // slot. - let compiled = compile(dag, &[], ids).expect("the executor compiles the plan"); - let instance = compiled.instantiate(Arc::new(store)).unwrap(); - block_on(async { - // The run's scope (ingestion window or query time) and limits. - let context = RunContext::new(scope, Limits::default()).unwrap(); - // One output stream per id. - let outputs = instance.execute(compiled.roots(), context).unwrap(); - futures::future::join_all(outputs.into_iter().map(|mut output| async move { - // Its batches, collected. - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.expect("the query runs")).clone()); - } - batches - })) - .await - }) -} - /// The series identity column's text: the canonical JSON of every label /// ([`PROMQL_SERIES_IDENTITY`]). pub fn series_identity(labels: &BTreeMap) -> String { @@ -188,3 +162,78 @@ pub fn answers(batches: &[Batch]) -> Vec { answers.sort_by(|a, b| a.0.cmp(&b.0)); answers } + +/// The node that answers each query of `plan`, in workload order. +pub fn query_roots(plan: &InstallablePlan) -> Vec { + plan.dag.roots.iter().map(|&root| root as u64).collect() +} + +/// Compile `ids` of `dag` and run them in `scope` over `store` (usually +/// [`time_series_tables`]), as a runtime does with the executor: each id's +/// output batches. +pub fn run( + dag: &PhysicalASAPDAG, + store: impl DataStore + 'static, + ids: &[u64], + scope: Scope, +) -> Vec> { + // The executor's plan for `ids`; every `Scan` it reaches is a slot of + // `store`. + let compiled = compile(dag, &[], ids).expect("the executor compiles the plan"); + let instance = compiled.instantiate(Arc::new(store)).unwrap(); + block_on(async { + // The run's scope (ingestion window or query time) and limits. + let context = RunContext::new(scope, Limits::default()).unwrap(); + // One output stream per id. + let outputs = instance.execute(compiled.roots(), context).unwrap(); + futures::future::join_all(outputs.into_iter().map(|mut output| async move { + // Its batches, collected. + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch.unwrap()).clone()); + } + batches + })) + .await + }) +} + +/// Asserts `answered` matches `expected`, label set for label set, each +/// value within absolute error `tolerance`: 10.04 is near 10 at 0.05. +pub fn assert_answers_near(answered: &[Answer], expected: &[Answer], tolerance: f64) { + assert_answers_within(answered, expected, |_| tolerance); +} + +/// Asserts `answered` matches `expected`, label set for label set, each +/// value within relative error `relative_tolerance` of its expected value: +/// 1000.5 is near 1000 at 0.001. +pub fn assert_answers_near_relative( + answered: &[Answer], + expected: &[Answer], + relative_tolerance: f64, +) { + assert_answers_within(answered, expected, |expected_value| { + relative_tolerance * expected_value.abs() + }); +} + +/// Asserts `answered` matches `expected`, label set for label set, each +/// value within `allowed_error(expected value)`. +fn assert_answers_within( + answered: &[Answer], + expected: &[Answer], + allowed_error: impl Fn(f64) -> f64, +) { + assert_eq!( + answered.len(), + expected.len(), + "{answered:?} vs {expected:?}" + ); + for (answer, expected_answer) in answered.iter().zip(expected) { + assert_eq!(answer.0, expected_answer.0, "{answered:?} vs {expected:?}"); + assert!( + (answer.1 - expected_answer.1).abs() <= allowed_error(expected_answer.1), + "{answered:?} vs {expected:?}" + ); + } +} diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index 55692ec4f..a4fbca18c 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -58,14 +58,12 @@ pub enum QueryLanguage { } impl QueryLanguage { - /// Whether this language's queries read time series, whose rows the - /// planner keys by each series' full label set: the - /// [`PROMQL_SERIES_IDENTITY`](crate::ir::schema::PROMQL_SERIES_IDENTITY) - /// column, added to every root before Stage 0. PromQL and MetricsQL - /// do: `rate(requests_total[5m])` keeps one rate per series, each - /// series `{"__name__":"requests_total","instance":"a","job":"j1"}` - /// as one identity. SQL reads tables, whose rows have no such identity. - pub fn has_series_identity(&self) -> bool { + /// Whether this language's queries read time series rather than + /// tables. PromQL and MetricsQL read time series: + /// `rate(requests_total[5m])` reads each `requests_total` series, such + /// as `{instance="a", job="j1"}`. SQL, DataFusion and ElasticDSL read + /// tables. + pub fn reads_time_series(&self) -> bool { matches!(self, Self::PromQL | Self::MetricsQL) } } @@ -902,10 +900,10 @@ mod tests { } } - /// The two time-series languages carry the series identity; the table - /// languages do not. + /// PromQL and MetricsQL read time series; the other languages read + /// tables. #[test] - fn only_time_series_languages_have_series_identity() { + fn only_promql_and_metricsql_read_time_series() { let cases = [ (QueryLanguage::PromQL, true), (QueryLanguage::MetricsQL, true), @@ -914,7 +912,7 @@ mod tests { (QueryLanguage::ElasticDSL, false), ]; for (language, expected) in cases { - assert_eq!(language.has_series_identity(), expected, "{language:?}"); + assert_eq!(language.reads_time_series(), expected, "{language:?}"); } }