From 697178e05d81668355d8d59dfd80f4015391388a Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:36:08 +0000 Subject: [PATCH] test(planner): counter rate and increase answers, installed and at query time Hand-worked answers of rate and increase over a counter, per series, per job and in total, on deployments that keep raw data and that do not, answered two ways: as installed (each tumbling-window chain built window by window in ingestion runs and read from its materialized view) and with the whole plan run in the query run from the raw samples. The query-run test fails today: a chain placed at ingestion time and run in the query run builds only its newest 10 s window, so a counter's per-series rows are dropped (fewer than two samples) and the grouped query answers no rows. --- crates/planner/tests/metricsql_workloads.rs | 2 +- crates/planner/tests/reduce_aggregates.rs | 1 + crates/planner/tests/runtime_install.rs | 200 +++++++++++++++++++- crates/planner/tests/time_series/mod.rs | 30 ++- 4 files changed, 222 insertions(+), 11 deletions(-) diff --git a/crates/planner/tests/metricsql_workloads.rs b/crates/planner/tests/metricsql_workloads.rs index c95de25e5..2d2dd1707 100644 --- a/crates/planner/tests/metricsql_workloads.rs +++ b/crates/planner/tests/metricsql_workloads.rs @@ -109,7 +109,7 @@ fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec BTreeMap { labels } -/// Each query root's answers of `plan` at [`EVALUATION_MS`] over `samples`. +/// Each query root's answers of `plan` at [`EVALUATION_MS`] over `samples`, +/// every node run in the query run (no materialized view bound): a +/// tumbling-window chain is built there from the raw samples. fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec> { let query = Scope::Query { evaluation_time_ms: EVALUATION_MS, @@ -454,6 +458,7 @@ fn query_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec InstallablePlan { + let workload: PlanningWorkload = serde_json::from_str(&format!( + r#"{{"query_workload": {{"language": "promql", "repeating_queries": [{{ + "query": {query:?}, + "demand": {{"fixed_interval_at": {{"interval": 10000, "evaluation_phase": 0}}}}, + "requirements": {{"accuracy": {{"max_error": 0.01}}}}, + "predictability": {{"predictable": {{"known_at": null}}}}}}]}}, + "data_workload": {{"arrival": {{"ingesting": {{"rows_per_second": 1000.0}}}}, + "sources": {{"requests_total": {{"series_count": 1000, + "metric_type": "counter", "sample_interval": 1000}}}}}}}}"# + )) + .unwrap(); + let capabilities = DeploymentCapabilities { + raw_data_retained, + ..DeploymentCapabilities::default() + }; + let models = PlanningModels::builtin().with_deployment_capabilities(&capabilities); + plan_installable(UserInput::new(&workload, FrontendInput::Promql, models)) + .await + .unwrap_or_else(|e| panic!("{query} (raw data retained: {raw_data_retained}): {e}")) + .0 +} + +/// Four `requests_total` series (instances `a`..`d`, jobs `j1` and `j2`), +/// sampled every 15 s from 8 minutes before [`EVALUATION_MS`] through it: +/// series `i` (0-based) counts `(i + 1) * step` at step 0..=32. +fn counter_samples() -> Vec { + let series = [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")]; + let mut samples = Vec::new(); + for (index, (instance, job)) in series.iter().enumerate() { + for step in 0..=32i64 { + samples.push(sample( + "requests_total", + &[("instance", instance), ("job", job)], + EVALUATION_MS - 480_000 + step * 15_000, + (index as f64 + 1.0) * step as f64, + )); + } + } + samples +} + +/// The counter queries over [`counter_samples`] and their answers at T, +/// worked out by hand. Over `[5m]` each series has the 20 samples of steps +/// 13..=32 (T - 285 s .. T), `(i + 1) * 19` apart; the 15 s gap to the +/// window's start is within 1.1 sample intervals, so Prometheus +/// extrapolates the increase to the whole 300 s: `(i + 1) * 20`, and the +/// rate is that over 300 s. Job `j1` is series 0 and 1, `j2` series 2 and +/// 3: increases 20 + 40 = 60 and 60 + 80 = 140, rates 0.2 and 0.4667. +fn counter_queries() -> Vec<(&'static str, Vec)> { + let series = |instance: &str, job: &str| labels(&[("instance", instance), ("job", job)]); + let job = |job: &str| labels(&[("job", job)]); + let per_series = |scale: f64| { + [("a", "j1"), ("b", "j1"), ("c", "j2"), ("d", "j2")] + .iter() + .enumerate() + .map(|(i, (instance, j))| (series(instance, j), (i as f64 + 1.0) * 20.0 * scale)) + .collect() + }; + let rate = 1.0 / 300.0; + vec![ + ("increase(requests_total[5m])", per_series(1.0)), + ("rate(requests_total[5m])", per_series(rate)), + ( + "sum by (job)(increase(requests_total[5m]))", + vec![(job("j1"), 60.0), (job("j2"), 140.0)], + ), + ( + "sum by (job)(rate(requests_total[5m]))", + vec![(job("j1"), 60.0 * rate), (job("j2"), 140.0 * rate)], + ), + ( + "sum(increase(requests_total[5m]))", + vec![(BTreeMap::new(), 200.0)], + ), + ( + "sum(rate(requests_total[5m]))", + vec![(BTreeMap::new(), 200.0 * rate)], + ), + ] +} + +/// Each query root's answers of `plan` at [`EVALUATION_MS`] as a runtime +/// that installed it answers: each materialized view (a tumbling-window +/// chain) is refreshed from `samples` in one ingestion run per tumbling +/// window its retention keeps, ending at T, and stored; the query run at +/// T reads those views (and the raw samples the plan reads at query time). +fn installed_answers(plan: &InstallablePlan, samples: &[RawDataSample]) -> Vec> { + let dag = &plan.dag; + // The raw samples, then each view's stored windows. + let mut store = time_series_tables(dag, samples); + let views = plan.materialized_views().unwrap(); + for view in &views { + let node = dag.nodes.iter().find(|n| n.id == view.node_id).unwrap(); + let Operator::ASAP(ASAPOp::SummaryTumblingWindowChain { + tumbling_window_width_ms, + .. + }) = node.payload + else { + panic!("only tumbling-window chains are refreshed here: {node:?}"); + }; + let width_ms = tumbling_window_width_ms as i64; + let windows = (view.retention_ms() as i64 / width_ms).max(1); + // The windows oldest first, each built in its own ingestion run. + for k in (0..windows).rev() { + let end = EVALUATION_MS - k * width_ms; + let ingestion = Scope::Ingestion { + window_start_ms: end - width_ms, + window_end_ms: end, + }; + let batches = run( + dag, + time_series_tables(dag, samples), + &[], + &[view.node_id as u64], + ingestion, + ) + .remove(0); + let rows = batches.iter().flat_map(|batch| batch.to_rows()).collect(); + let schema = Arc::new(node.output_schema.clone()); + let window = Batch::try_new(schema, rows).unwrap(); + store + .append_materialized_view(view, (end - width_ms, end), vec![window]) + .unwrap(); + } + } + let query = Scope::Query { + evaluation_time_ms: EVALUATION_MS, + }; + run(dag, store, &views, &query_roots(plan), query) + .iter() + .map(|batches| answers(batches)) + .collect() +} + +/// Each counter query of [`counter_queries`], planned on a deployment that +/// keeps raw data and on one that does not, answered by `answer` over +/// [`counter_samples`]: the queries whose answers differ from the +/// hand-worked ones (relative error 1e-9). +async fn counter_query_mismatches( + answer: fn(&InstallablePlan, &[RawDataSample]) -> Vec>, +) -> Vec { + let samples = counter_samples(); + let mut mismatches = Vec::new(); + for raw_data_retained in [true, false] { + for (query, expected) in counter_queries() { + let plan = plan_counter_query(query, raw_data_retained).await; + let answered = answer(&plan, &samples).remove(0); + let matches = answered.len() == expected.len() + && answered.iter().zip(&expected).all(|(answer, expected)| { + answer.0 == expected.0 && (answer.1 - expected.1).abs() <= 1e-9 * expected.1 + }); + if !matches { + mismatches.push(format!( + "{query} (raw data retained: {raw_data_retained}): {answered:?}, \ + expected {expected:?}" + )); + } + } + } + mismatches +} + +/// Counter `rate` and `increase`, per series, per job and in total, answer +/// the hand-worked values as installed: each tumbling-window chain built +/// window by window in ingestion runs and read from its materialized view +/// at query time. +#[tokio::test] +async fn counter_rates_and_increases_answer_from_views_built_at_ingestion_time() { + let mismatches = counter_query_mismatches(installed_answers).await; + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} + +/// Counter `rate` and `increase`, per series, per job and in total, answer +/// the hand-worked values when the whole plan runs in the query run from +/// the raw samples, no materialized view bound: a tumbling-window chain +/// placed at ingestion time is then built in the query run, every window +/// its readers read, not only the newest. +#[tokio::test] +async fn counter_rates_and_increases_answer_at_query_time_from_raw_samples() { + let mismatches = counter_query_mismatches(query_answers).await; + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} diff --git a/crates/planner/tests/time_series/mod.rs b/crates/planner/tests/time_series/mod.rs index 03617ef4d..c8dd4629c 100644 --- a/crates/planner/tests/time_series/mod.rs +++ b/crates/planner/tests/time_series/mod.rs @@ -13,7 +13,7 @@ use asap_executor::physical_planner::compile; use asap_executor::runtime::{Limits, RunContext, Scope}; use asap_executor::sources::{DataStore, InMemoryDataStore}; use asap_executor::values::{Batch, Value}; -use asap_planner::InstallablePlan; +use asap_planner::{InstallablePlan, MaterializedView}; use asap_types::ir::operator::Source; use asap_types::ir::schema::{DataType, FieldDataType, Schema, PROMQL_SERIES_IDENTITY}; use asap_types::ir::NonASAPOp; @@ -169,22 +169,38 @@ pub fn query_roots(plan: &InstallablePlan) -> Vec { } /// Compile `ids` of `dag` and run them in `scope` over `store` (usually -/// [`time_series_tables`]), as a runtime does with the executor: each id's -/// output batches. +/// [`time_series_tables`], plus the stored states of `views`), as a runtime +/// does with the executor: each id's output batches. Each of `views` is +/// bound as its materialized view slot (nothing below it compiled); pass +/// `&[]` to compute every node from raw data. pub fn run( dag: &PhysicalASAPDAG, store: impl DataStore + 'static, + views: &[MaterializedView], ids: &[u64], scope: Scope, ) -> Vec> { - // The executor's plan for `ids`; every `Scan` it reaches is a slot of - // `store`. - let compiled = compile(dag, &[], ids).expect("the executor compiles the plan"); + // The executor's plan for `ids`; every `Scan` it reaches and every view + // in `views` is a slot of `store`. + let compiled = compile(dag, views, ids).expect("the executor compiles the plan"); let instance = compiled.instantiate(Arc::new(store)).unwrap(); + let ingestion = matches!(scope, Scope::Ingestion { .. }); block_on(async { // The run's scope (ingestion window or query time) and limits. let context = RunContext::new(scope, Limits::default()).unwrap(); - // One output stream per id. + if ingestion { + // An ingestion run: every id's batches, for the runtime to store. + let outputs = instance + .execute_ingestion(compiled.roots(), context) + .unwrap() + .await + .unwrap(); + return outputs + .into_iter() + .map(|batches| batches.iter().map(|batch| (**batch).clone()).collect()) + .collect(); + } + // A query run: one output stream per id. let outputs = instance.execute(compiled.roots(), context).unwrap(); futures::future::join_all(outputs.into_iter().map(|mut output| async move { // Its batches, collected.