diff --git a/crates/planner/src/run.rs b/crates/planner/src/run.rs index 1f69236bc..2caf7e752 100644 --- a/crates/planner/src/run.rs +++ b/crates/planner/src/run.rs @@ -17,7 +17,7 @@ use asap_types::ir::schema::series_identity::{with_promql_series_identity, Serie use asap_types::ir::{OperatorNode, QueryRoot}; use asap_types::plan::physical_asap_dag::PhysicalASAPDAG; use asap_types::stages::{LogicalASAPMemoCandidates, Memo, PhysicalASAPMemoCandidates}; -use asap_types::workload::{DataWorkload, QueryLanguage, QueryWorkloadEntry}; +use asap_types::workload::{DataWorkload, QueryWorkloadEntry}; use crate::models::PlanningModels; use crate::pass::{OptimizationInput, OptimizationPass, OptimizeError, PlanOutput}; @@ -169,12 +169,13 @@ fn elapsed_ms(started: Instant) -> f64 { } /// The planner pipeline behind the optimization-pass interface: the -/// workload's roots in entry order, PromQL roots with each series' full -/// identity as a column (the row representation per-series state needs at -/// runtime; a root with no such representation is planned over its labels -/// alone), through [`run_planning`] with every entry's accuracy target. The -/// output is the plan Stage 3 built; its `Selection` report is -/// [`run_planning`]'s. +/// workload's roots in entry order, those of a time-series language +/// ([`asap_types::workload::QueryLanguage::reads_time_series`]: PromQL and +/// MetricsQL) with each series' full identity as a column (the row +/// representation per-series state needs at runtime; a root with no such +/// representation is planned over its labels alone), through +/// [`run_planning`] with every entry's accuracy target. The output is the +/// plan Stage 3 built; its `Selection` report is [`run_planning`]'s. #[derive(Debug, Default, Clone, Copy)] pub struct Stage1ToStage3Pass; @@ -185,13 +186,18 @@ impl OptimizationPass for Stage1ToStage3Pass { fn optimize(&self, input: OptimizationInput<'_>) -> Result { let workload = input.workload; - let promql = workload.query_workload().language == QueryLanguage::PromQL; + // Time-series rows are keyed by each series' full label set: the + // `PROMQL_SERIES_IDENTITY` column, added to every root before Stage + // 0. `rate(requests_total[5m])` keeps one rate per series, each + // series `{"__name__":"requests_total","instance":"a","job":"j1"}` + // one identity. Table rows have no such identity. + let series_identity = workload.query_workload().language.reads_time_series(); let mut roots: Vec<(usize, QueryRoot)> = workload .operator_indices() .iter() .zip(workload.exprs()) .map(|(&index, node)| { - let node = match promql { + let node = match series_identity { true => promql_root(node, with_promql_series_identity(node))?, false => Rc::clone(node), }; @@ -221,7 +227,7 @@ impl OptimizationPass for Stage1ToStage3Pass { } } -/// The PromQL root `node` planned: `realized`, its series identity +/// The PromQL or MetricsQL root `node` planned: `realized`, its series identity /// realization (`with_promql_series_identity`), or `node` itself, over its /// label columns, when the realization does not apply to one of its /// operators (`limitk(2, m)`'s `PromqlSeriesSample`, `info(m)`'s diff --git a/crates/planner/tests/main.rs b/crates/planner/tests/main.rs index d16b381fa..d72a470e7 100644 --- a/crates/planner/tests/main.rs +++ b/crates/planner/tests/main.rs @@ -7,6 +7,7 @@ mod time_series; mod columnar_batches; mod e2e_plan; mod exact_only_reads; +mod metricsql_workloads; mod reduce_aggregates; mod runtime_install; mod sql_table_scans; diff --git a/crates/planner/tests/metricsql_workloads.rs b/crates/planner/tests/metricsql_workloads.rs new file mode 100644 index 000000000..c95de25e5 --- /dev/null +++ b/crates/planner/tests/metricsql_workloads.rs @@ -0,0 +1,216 @@ +//! A MetricsQL workload plans, installs and answers as its PromQL form +//! does: both languages read time series, so each query root carries every +//! series' full identity as a column before Stage 0, whichever of the two +//! frontends lowered it. +//! +//! The workload: 1,000 series per metric sampled every second, each query +//! every 10 s within relative error 0.01. Without the series identity a +//! MetricsQL `rate`, `increase` or `max_over_time` root has no valid plan +//! ("per-entity summary requires complete source identity") and `sum` over +//! an instant selector selects a plan whose output type differs from the +//! query's. + +use std::collections::BTreeMap; + +use asap_executor::runtime::Scope; +use asap_planner::{ + plan_installable, FrontendInput, InstallablePlan, PlanningModels, PlanningWorkload, UserInput, +}; + +use crate::time_series::{ + answers, assert_answers_near_relative, query_roots, run, sample, time_series_tables, Answer, + RawDataSample, +}; + +/// When every query is evaluated: the end of the first hour. +const EVALUATION_MS: i64 = 3_600_000; + +/// Queries both languages write the same way, each over one of the two +/// metrics [`workload`] declares. +const QUERIES: [&str; 4] = [ + "sum by (job)(rate(requests_total[5m]))", + "increase(requests_total[5m])", + "max_over_time(latency[5m])", + r#"sum(latency{job="j1"})"#, +]; + +/// `query` written in `language` ("promql" or "metricsql"), every 10 s +/// within relative error 0.01, over 1,000 `requests_total` counter series +/// and 1,000 `latency` gauge series, each sampled every second. +fn workload(language: &str, query: &str) -> PlanningWorkload { + serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "{language}", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.01}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 2000.0}}}}, + "sources": {{ + "requests_total": {{"series_count": 1000, "metric_type": "counter", + "sample_interval": 1000}}, + "latency": {{"series_count": 1000, "metric_type": "gauge", + "sample_interval": 1000}}}}}}}}"# + )) + .unwrap() +} + +/// The built-in plan of `query` written in MetricsQL, or the error the +/// Planner returned. +async fn plan_metricsql(query: &str) -> Result { + let workload = workload("metricsql", query); + plan_installable(UserInput::new( + &workload, + FrontendInput::Metricsql, + PlanningModels::builtin(), + )) + .await + .map(|(plan, _report)| plan) + .map_err(|e| e.to_string()) +} + +/// The built-in plan of `query` written in PromQL. +async fn plan_promql(query: &str) -> InstallablePlan { + let workload = workload("promql", query); + plan_installable(UserInput::new( + &workload, + FrontendInput::Promql, + PlanningModels::builtin(), + )) + .await + .unwrap_or_else(|e| panic!("{query} as PromQL: {e}")) + .0 +} + +/// Four series of each metric (instances `a`..`d`, jobs `j1` and `j2`), +/// sampled every 15 s from 8 minutes before [`EVALUATION_MS`] through it: +/// `requests_total` grows by `index + 1` per sample, and `latency` cycles +/// through `index * 10 + step % 7`. +fn samples() -> Vec { + let series = [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")]; + let mut samples = Vec::new(); + for (index, (instance, job)) in series.iter().enumerate() { + for step in 0..=32i64 { + let timestamp_ms = EVALUATION_MS - 480_000 + step * 15_000; + let count = (index as f64 + 1.0) * step as f64; + let latency = index as f64 * 10.0 + (step % 7) as f64; + let labels = [("instance", *instance), ("job", *job)]; + samples.push(sample("requests_total", &labels, timestamp_ms, count)); + samples.push(sample("latency", &labels, timestamp_ms, latency)); + } + } + samples +} + +/// The answers of `plan`'s one query at [`EVALUATION_MS`] over `samples`, +/// sorted by labels: the executor compiles the plan and runs its root at +/// query time over the raw samples. +fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec { + let scope = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + let store = time_series_tables(&plan.dag, samples); + answers(&run(&plan.dag, store, &query_roots(plan), scope)[0]) +} + +/// Each query written in MetricsQL plans and installs: the executor +/// compiles its plan. +#[tokio::test] +async fn metricsql_queries_plan_and_install() { + let mut failures = Vec::new(); + for query in QUERIES { + let installed = plan_metricsql(query) + .await + .and_then(|plan| plan.check_compiles()); + if let Err(error) = installed { + failures.push(format!("{query}: {error}")); + } + } + assert!(failures.is_empty(), "{failures:#?}"); +} + +/// Each query's MetricsQL plan is its PromQL plan: the same selected DAG, +/// node for node. +#[tokio::test] +async fn metricsql_queries_select_the_promql_plan() { + for query in QUERIES { + let metricsql = plan_metricsql(query) + .await + .unwrap_or_else(|e| panic!("{query} as MetricsQL: {e}")); + let promql = plan_promql(query).await; + assert_eq!(metricsql.dag, promql.dag, "{query}"); + } +} + +/// Each query's MetricsQL plan answers what its PromQL plan answers on the +/// same samples, and both match the answers worked out by hand from +/// [`samples`] at T. `increase` over 5 m sees 20 samples per series, +/// `(index + 1) * 19` apart over 285 s, extrapolated to the 300 s window: +/// `(index + 1) * 20`. `max_over_time` over 5 m sees steps 13..=32, whose +/// largest `step % 7` is 6. `sum(latency{job="j1"})` reads each `j1` +/// series' sample at T (step 32, `32 % 7` = 4): 4 + 14 = 18. The grouped +/// counter query answers no rows in either language today (issue #1338), +/// so it checks only that the two languages agree. +#[tokio::test] +async fn metricsql_queries_answer_as_promql_does() { + let series = |instance: &str, job: &str| { + BTreeMap::from([ + ("instance".to_string(), instance.to_string()), + ("job".to_string(), job.to_string()), + ]) + }; + let per_series = |values: [f64; 4]| -> Option> { + let names = [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")]; + Some( + names + .iter() + .zip(values) + .map(|(&(i, j), v)| (series(i, j), v)) + .collect(), + ) + }; + // Each query's hand-worked answers; `None` where only the languages' + // agreement is checked. + let expected: [Option>; 4] = [ + None, + per_series([20.0, 40.0, 60.0, 80.0]), + per_series([6.0, 16.0, 26.0, 36.0]), + Some(vec![(BTreeMap::new(), 18.0)]), + ]; + let samples = samples(); + for (query, expected) in QUERIES.into_iter().zip(expected) { + let metricsql = plan_metricsql(query) + .await + .unwrap_or_else(|e| panic!("{query} as MetricsQL: {e}")); + let metricsql_answers = query_answers(&metricsql, &samples); + assert_eq!( + metricsql_answers, + query_answers(&plan_promql(query).await, &samples), + "{query}" + ); + let Some(expected) = expected else { continue }; + assert_answers_near_relative(&metricsql_answers, &expected, 1e-9); + } +} + +/// `avg by (job)(latency)` in MetricsQL averages each job's series at T, as +/// its PromQL form does: `j1` holds series `a` (latency 4 at step 32) and +/// `b` (14), so 9; `j2` holds `c` (24) and `d` (34), so 29. A plan whose +/// rows lack the series identity plans and runs without any error, yet +/// answers a different average, so this checks the values, not only that +/// the query plans. +#[tokio::test] +async fn metricsql_grouped_average_averages_every_series_of_each_group() { + let query = "avg by (job)(latency)"; + let job = |name: &str| BTreeMap::from([("job".to_string(), name.to_string())]); + let expected = vec![(job("j1"), 9.0), (job("j2"), 29.0)]; + let samples = samples(); + let metricsql = plan_metricsql(query) + .await + .unwrap_or_else(|e| panic!("{query} as MetricsQL: {e}")); + let metricsql_answers = query_answers(&metricsql, &samples); + assert_answers_near_relative(&metricsql_answers, &expected, 1e-9); + assert_eq!( + metricsql_answers, + query_answers(&plan_promql(query).await, &samples) + ); +} diff --git a/crates/planner/tests/runtime_install.rs b/crates/planner/tests/runtime_install.rs index d4d85ebe8..6f1d6348a 100644 --- a/crates/planner/tests/runtime_install.rs +++ b/crates/planner/tests/runtime_install.rs @@ -4,25 +4,20 @@ //! the compiler checks that no internal is used (CI runs //! `cargo check -p asap-planner --tests --locked` on its own for that). -use std::cell::Cell; use std::collections::{BTreeMap, BTreeSet}; use crate::time_series; -use asap_executor::runtime::{OutputStream, RunContext, Scope}; -use asap_executor::sources::{ - DataStore, InMemoryDataStore, MaterializedViewRequest, RawDataRequest, -}; -use asap_executor::values::{Batch, Value}; -use asap_executor::Error as ExecutorError; +use asap_executor::runtime::Scope; use asap_planner::{ plan_installable, CostInputs, DeploymentCapabilities, FrontendInput, InstallablePlan, - MaterializedView, OperatorKind, PlanningModels, PlanningWorkload, SelectedPlanReport, - UserInput, + OperatorKind, PlanningModels, PlanningWorkload, SelectedPlanReport, UserInput, }; -use asap_types::ir::operator::Source; use asap_types::ir::{ASAPOp, NonASAPOp, Operator}; -use time_series::{run, sample, series_labels, time_series_tables, RawDataSample}; +use time_series::{ + answers, assert_answers_near, query_roots, run, sample, time_series_tables, Answer, + RawDataSample, +}; /// Example 4b's evaluation time: the end of the 60-minute window. const EVALUATION_MS: i64 = 3_600_000; @@ -60,57 +55,6 @@ fn workload_every(queries: &[(&str, u64)]) -> PlanningWorkload { .unwrap() } -thread_local! { - /// How many `read_raw_data` calls the runs on this thread made. - static RAW_DATA_READS: Cell = const { Cell::new(0) }; -} - -/// A store counting its `read_raw_data` calls in [`RAW_DATA_READS`]. -struct CountingStore(InMemoryDataStore); - -impl DataStore for CountingStore { - fn read_raw_data( - &self, - source: &Source, - request: &RawDataRequest, - context: RunContext, - ) -> Result, ExecutorError> { - RAW_DATA_READS.with(|reads| reads.set(reads.get() + 1)); - self.0.read_raw_data(source, request, context) - } - - fn read_materialized_view( - &self, - view: &MaterializedView, - request: &MaterializedViewRequest, - context: RunContext, - ) -> Result, ExecutorError> { - self.0.read_materialized_view(view, request, context) - } -} - -/// One answer row: its series labels and its value. -type Answer = (BTreeMap, f64); - -/// Each answer row as its series labels and value, sorted by labels. -fn answers(batches: &[Batch]) -> Vec { - let mut answers: Vec<_> = batches - .iter() - .flat_map(|batch| { - batch.to_rows().into_iter().map(|row| { - let labels = series_labels(batch.schema(), &row); - let value = row.iter().find_map(|v| match v { - Value::Float64(v) => Some(*v), - _ => None, - }); - (labels, value.expect("a Float64 answer")) - }) - }) - .collect(); - answers.sort_by(|a, b| a.0.cmp(&b.0)); - answers -} - /// The plan for `queries` on a deployment that keeps no raw samples, and /// Stage 3's report of the selection it made (which the plan file does not /// carry). @@ -148,11 +92,6 @@ async fn plan_with_report_for( .unwrap() } -/// The node that answers each query of `plan`, in workload order. -fn query_roots(plan: &InstallablePlan) -> Vec { - plan.dag.roots.iter().map(|&root| root as u64).collect() -} - /// The plan file carries Stage 3's predicted usage of the selected plan, so /// a runtime reports it against what it measures without planning again: /// for Example 4b's tumbling windows maintained at ingestion time, its nodes' costs @@ -296,7 +235,7 @@ async fn an_instant_selector_reads_the_latest_sample_per_series() { let query = Scope::Query { evaluation_time_ms: EVALUATION_MS, }; - let store = CountingStore(time_series_tables(dag, &samples)); + let store = time_series_tables(dag, &samples); let answered = answers(&run(dag, store, &query_roots(&plan), query)[0]); assert_eq!(answered.len(), 1); assert_eq!(answered[0].1, 110.0); @@ -512,28 +451,15 @@ fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec Vec> { - // The executor's plan for `ids`; every `Scan` it reaches is a raw data - // slot. - let compiled = compile(dag, &[], ids).expect("the executor compiles the plan"); - let instance = compiled.instantiate(Arc::new(store)).unwrap(); - block_on(async { - // The run's scope (ingestion window or query time) and limits. - let context = RunContext::new(scope, Limits::default()).unwrap(); - // One output stream per id. - let outputs = instance.execute(compiled.roots(), context).unwrap(); - futures::future::join_all(outputs.into_iter().map(|mut output| async move { - // Its batches, collected. - let mut batches = Vec::new(); - while let Some(batch) = output.next().await { - batches.push((*batch.expect("the query runs")).clone()); - } - batches - })) - .await - }) -} - /// The series identity column's text: the canonical JSON of every label /// ([`PROMQL_SERIES_IDENTITY`]). pub fn series_identity(labels: &BTreeMap) -> String { @@ -166,3 +140,100 @@ pub fn series_labels(schema: &Schema, row: &[Value]) -> BTreeMap } labels } + +/// One answer row: its series labels and its value. +pub type Answer = (BTreeMap, f64); + +/// Each answer row as its series labels and value, sorted by labels. +pub fn answers(batches: &[Batch]) -> Vec { + let mut answers: Vec<_> = batches + .iter() + .flat_map(|batch| { + batch.to_rows().into_iter().map(|row| { + let labels = series_labels(batch.schema(), &row); + let value = row.iter().find_map(|v| match v { + Value::Float64(v) => Some(*v), + _ => None, + }); + (labels, value.expect("a Float64 answer")) + }) + }) + .collect(); + answers.sort_by(|a, b| a.0.cmp(&b.0)); + answers +} + +/// The node that answers each query of `plan`, in workload order. +pub fn query_roots(plan: &InstallablePlan) -> Vec { + plan.dag.roots.iter().map(|&root| root as u64).collect() +} + +/// Compile `ids` of `dag` and run them in `scope` over `store` (usually +/// [`time_series_tables`]), as a runtime does with the executor: each id's +/// output batches. +pub fn run( + dag: &PhysicalASAPDAG, + store: impl DataStore + 'static, + ids: &[u64], + scope: Scope, +) -> Vec> { + // The executor's plan for `ids`; every `Scan` it reaches is a slot of + // `store`. + let compiled = compile(dag, &[], ids).expect("the executor compiles the plan"); + let instance = compiled.instantiate(Arc::new(store)).unwrap(); + block_on(async { + // The run's scope (ingestion window or query time) and limits. + let context = RunContext::new(scope, Limits::default()).unwrap(); + // One output stream per id. + let outputs = instance.execute(compiled.roots(), context).unwrap(); + futures::future::join_all(outputs.into_iter().map(|mut output| async move { + // Its batches, collected. + let mut batches = Vec::new(); + while let Some(batch) = output.next().await { + batches.push((*batch.unwrap()).clone()); + } + batches + })) + .await + }) +} + +/// Asserts `answered` matches `expected`, label set for label set, each +/// value within absolute error `tolerance`: 10.04 is near 10 at 0.05. +pub fn assert_answers_near(answered: &[Answer], expected: &[Answer], tolerance: f64) { + assert_answers_within(answered, expected, |_| tolerance); +} + +/// Asserts `answered` matches `expected`, label set for label set, each +/// value within relative error `relative_tolerance` of its expected value: +/// 1000.5 is near 1000 at 0.001. +pub fn assert_answers_near_relative( + answered: &[Answer], + expected: &[Answer], + relative_tolerance: f64, +) { + assert_answers_within(answered, expected, |expected_value| { + relative_tolerance * expected_value.abs() + }); +} + +/// Asserts `answered` matches `expected`, label set for label set, each +/// value within `allowed_error(expected value)`. +fn assert_answers_within( + answered: &[Answer], + expected: &[Answer], + allowed_error: impl Fn(f64) -> f64, +) { + assert_eq!( + answered.len(), + expected.len(), + "{answered:?} vs {expected:?}" + ); + for (answer, expected_answer) in answered.iter().zip(expected) { + assert_eq!(answer.0, expected_answer.0, "{answered:?} vs {expected:?}"); + assert!( + (answer.1 - expected_answer.1).abs() <= allowed_error(expected_answer.1), + "{answered:?} vs {expected:?}" + ); + } +} diff --git a/crates/types/src/workload/query.rs b/crates/types/src/workload/query.rs index 2e5c64d2f..a4fbca18c 100644 --- a/crates/types/src/workload/query.rs +++ b/crates/types/src/workload/query.rs @@ -57,6 +57,17 @@ pub enum QueryLanguage { ElasticDSL, } +impl QueryLanguage { + /// Whether this language's queries read time series rather than + /// tables. PromQL and MetricsQL read time series: + /// `rate(requests_total[5m])` reads each `requests_total` series, such + /// as `{instance="a", job="j1"}`. SQL, DataFusion and ElasticDSL read + /// tables. + pub fn reads_time_series(&self) -> bool { + matches!(self, Self::PromQL | Self::MetricsQL) + } +} + /// Independent accuracy and response-latency constraints attached to one /// query in the workload. /// @@ -889,6 +900,22 @@ mod tests { } } + /// PromQL and MetricsQL read time series; the other languages read + /// tables. + #[test] + fn only_promql_and_metricsql_read_time_series() { + let cases = [ + (QueryLanguage::PromQL, true), + (QueryLanguage::MetricsQL, true), + (QueryLanguage::SQL(SqlDialect::DataFusionSQL), false), + (QueryLanguage::DataFusion, false), + (QueryLanguage::ElasticDSL, false), + ]; + for (language, expected) in cases { + assert_eq!(language.reads_time_series(), expected, "{language:?}"); + } + } + #[test] fn struct_enum_variants_reject_unknown_fields() { assert!(