diff --git a/crates/planner/tests/metricsql_workloads.rs b/crates/planner/tests/metricsql_workloads.rs index c95de25e5..2d2dd1707 100644 --- a/crates/planner/tests/metricsql_workloads.rs +++ b/crates/planner/tests/metricsql_workloads.rs @@ -109,7 +109,7 @@ fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec BTreeMap { labels } -/// Each query root's answers of `plan` at [`EVALUATION_MS`] over `samples`. +/// Each query root's answers of `plan` at [`EVALUATION_MS`] over `samples`, +/// every node run in the query run (no materialized view bound): a +/// tumbling-window chain is built there from the raw samples. fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec> { let query = Scope::Query { evaluation_time_ms: EVALUATION_MS, @@ -454,6 +458,7 @@ fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec InstallablePlan { + let workload: PlanningWorkload = serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.01}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, + "sources": {{"requests_total": {{"series_count": 1000, + "metric_type": "counter", "sample_interval": 1000}}}}}}}}"# + )) + .unwrap(); + let capabilities = DeploymentCapabilities { + raw_data_retained, + ..DeploymentCapabilities::default() + }; + let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); + plan_installable(UserInput::new(&workload, FrontendInput::Promql, models)) + .await + .unwrap_or_else(|e| panic!("{query} (raw data retained: {raw_data_retained}): {e}")) + .0 +} + +/// Four `requests_total` series (instances `a`..`d`, jobs `j1` and `j2`), +/// sampled every 15 s from 8 minutes before [`EVALUATION_MS`] through it: +/// series `i` (0-based) counts `(i + 1) * step` at step 0..=32. +fn counter_samples() -> Vec { + let series = [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")]; + let mut samples = Vec::new(); + for (index, (instance, job)) in series.iter().enumerate() { + for step in 0..=32i64 { + samples.push(sample( + "requests_total", + &[("instance", instance), ("job", job)], + EVALUATION_MS - 480_000 + step * 15_000, + (index as f64 + 1.0) * step as f64, + )); + } + } + samples +} + +/// The counter queries over [`counter_samples`] and their answers at T, +/// worked out by hand. Over `[5m]` each series has the 20 samples of steps +/// 13..=32 (T - 285 s .. T), `(i + 1) * 19` apart; the 15 s gap to the +/// window's start is within 1.1 sample intervals, so Prometheus +/// extrapolates the increase to the whole 300 s: `(i + 1) * 20`, and the +/// rate is that over 300 s. Job `j1` is series 0 and 1, `j2` series 2 and +/// 3: increases 20 + 40 = 60 and 60 + 80 = 140, rates 0.2 and 0.4667. +fn counter_queries() -> Vec<(&'static str, Vec)> { + let series = |instance: &str, job: &str| labels(&[("instance", instance), ("job", job)]); + let job = |job: &str| labels(&[("job", job)]); + let per_series = |scale: f64| { + [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")] + .iter() + .enumerate() + .map(|(i, (instance, j))| (series(instance, j), (i as f64 + 1.0) * 20.0 * scale)) + .collect() + }; + let rate = 1.0 / 300.0; + vec![ + ("increase(requests_total[5m])", per_series(1.0)), + ("rate(requests_total[5m])", per_series(rate)), + ( + "sum by (job)(increase(requests_total[5m]))", + vec![(job("j1"), 60.0), (job("j2"), 140.0)], + ), + ( + "sum by (job)(rate(requests_total[5m]))", + vec![(job("j1"), 60.0 * rate), (job("j2"), 140.0 * rate)], + ), + ( + "sum(increase(requests_total[5m]))", + vec![(BTreeMap::new(), 200.0)], + ), + ( + "sum(rate(requests_total[5m]))", + vec![(BTreeMap::new(), 200.0 * rate)], + ), + ] +} + +/// Each query root's answers of `plan` at [`EVALUATION_MS`] as a runtime +/// that installed it answers: each materialized view (a tumbling-window +/// chain) is refreshed from `samples` in one ingestion run per tumbling +/// window its retention keeps, ending at T, and stored; the query run at +/// T reads those views (and the raw samples the plan reads at query time). +fn installed_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec> { + let dag = &plan.dag; + // The raw samples, then each view's stored windows. + let mut store = time_series_tables(dag, samples); + let views = plan.materialized_views().unwrap(); + for view in &views { + let node = dag.nodes.iter().find(|n| n.id == view.node_id).unwrap(); + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) = node.payload + else { + panic!("only tumbling-window chains are refreshed here: {node:?}"); + }; + let width_ms = tumbling_window_width_ms as i64; + let windows = (view.retention_ms() as i64 / width_ms).max(1); + // The windows oldest first, each built in its own ingestion run. + for k in (0..windows).rev() { + let end = EVALUATION_MS - k * width_ms; + let ingestion = Scope::Ingestion { + window_start_ms: end - width_ms, + window_end_ms: end, + }; + let batches = run( + dag, + time_series_tables(dag, samples), + &[], + &[view.node_id as u64], + ingestion, + ) + .remove(0); + let rows = batches.iter().flat_map(|batch| batch.to_rows()).collect(); + let schema = Arc::new(node.output_schema.clone()); + let window = Batch::try_new(schema, rows).unwrap(); + store + .append_materialized_view(view, (end - width_ms, end), vec![window]) + .unwrap(); + } + } + let query = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + run(dag, store, &views, &query_roots(plan), query) + .iter() + .map(|batches| answers(batches)) + .collect() +} + +/// Each counter query of [`counter_queries`], planned on a deployment that +/// keeps raw data and on one that does not, answered by `answer` over +/// [`counter_samples`]: the queries whose answers differ from the +/// hand-worked ones (relative error 1e-9). +async fn counter_query_mismatches( + answer: fn(&InstallablePlan, &[RawDataSample]) -> Vec>, +) -> Vec { + let samples = counter_samples(); + let mut mismatches = Vec::new(); + for raw_data_retained in [true, false] { + for (query, expected) in counter_queries() { + let plan = plan_counter_query(query, raw_data_retained).await; + let answered = answer(&plan, &samples).remove(0); + let matches = answered.len() == expected.len() + && answered.iter().zip(&expected).all(|(answer, expected)| { + answer.0 == expected.0 && (answer.1 - expected.1).abs() <= 1e-9 * expected.1 + }); + if !matches { + mismatches.push(format!( + "{query} (raw data retained: {raw_data_retained}): {answered:?}, \ + expected {expected:?}" + )); + } + } + } + mismatches +} + +/// Counter `rate` and `increase`, per series, per job and in total, answer +/// the hand-worked values as installed: each tumbling-window chain built +/// window by window in ingestion runs and read from its materialized view +/// at query time. +#[tokio::test] +async fn counter_rates_and_increases_answer_from_views_built_at_ingestion_time() { + let mismatches = counter_query_mismatches(installed_answers).await; + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} + +/// Counter `rate` and `increase`, per series, per job and in total, answer +/// the hand-worked values when the whole plan runs in the query run from +/// the raw samples, no materialized view bound: a tumbling-window chain +/// placed at ingestion time is then built in the query run, every window +/// its readers read, not only the newest. +#[tokio::test] +async fn counter_rates_and_increases_answer_at_query_time_from_raw_samples() { + let mismatches = counter_query_mismatches(query_answers).await; + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} diff --git a/crates/planner/tests/time_series/mod.rs b/crates/planner/tests/time_series/mod.rs index 03617ef4d..c8dd4629c 100644 --- a/crates/planner/tests/time_series/mod.rs +++ b/crates/planner/tests/time_series/mod.rs @@ -13,7 +13,7 @@ use asap_executor::physical_planner::compile; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::sources::{DataStore, InMemoryDataStore}; use asap_executor::values::{Batch, Value}; -use asap_planner::InstallablePlan; +use asap_planner::{InstallablePlan, MaterializedView}; use asap_types::ir::operator::Source; use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; use asap_types::ir::NonASAPOp; @@ -169,22 +169,38 @@ pub fn query_roots(plan: &InstallablePlan) -> Vec { } /// Compile `ids` of `dag` and run them in `scope` over `store` (usually -/// [`time_series_tables`]), as a runtime does with the executor: each id's -/// output batches. +/// [`time_series_tables`], plus the stored states of `views`), as a runtime +/// does with the executor: each id's output batches. Each of `views` is +/// bound as its materialized view slot (nothing below it compiled); pass +/// `&[]` to compute every node from raw data. pub fn run( dag: &PhysicalASAPDAG, store: impl DataStore + 'static, + views: &[MaterializedView], ids: &[u64], scope: Scope, ) -> Vec> { - // The executor's plan for `ids`; every `Scan` it reaches is a slot of - // `store`. - let compiled = compile(dag, &[], ids).expect("the executor compiles the plan"); + // The executor's plan for `ids`; every `Scan` it reaches and every view + // in `views` is a slot of `store`. + let compiled = compile(dag, views, ids).expect("the executor compiles the plan"); let instance = compiled.instantiate(Arc::new(store)).unwrap(); + let ingestion = matches!(scope, Scope::Ingestion { .. }); block_on(async { // The run's scope (ingestion window or query time) and limits. let context = RunContext::new(scope, Limits::default()).unwrap(); - // One output stream per id. + if ingestion { + // An ingestion run: every id's batches, for the runtime to store. + let outputs = instance + .execute_ingestion(compiled.roots(), context) + .unwrap() + .await + .unwrap(); + return outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect(); + } + // A query run: one output stream per id. let outputs = instance.execute(compiled.roots(), context).unwrap(); futures::future::join_all(outputs.into_iter().map(|mut output| async move { // Its batches, collected.